2024-11-09 21:20:53,288 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-09 21:20:53,303 main DEBUG Took 0.012850 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-09 21:20:53,303 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-09 21:20:53,304 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-09 21:20:53,305 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-09 21:20:53,306 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 21:20:53,314 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-09 21:20:53,329 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 21:20:53,331 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 21:20:53,332 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 21:20:53,332 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 21:20:53,333 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 21:20:53,333 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 21:20:53,334 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 21:20:53,335 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 21:20:53,335 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 21:20:53,336 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 21:20:53,337 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 21:20:53,337 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 21:20:53,338 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 21:20:53,338 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 21:20:53,339 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 21:20:53,339 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 21:20:53,340 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 21:20:53,340 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 21:20:53,341 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 21:20:53,341 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 21:20:53,342 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 21:20:53,342 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 21:20:53,343 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 21:20:53,343 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-09 21:20:53,344 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 21:20:53,344 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-09 21:20:53,346 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-09 21:20:53,348 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-09 21:20:53,350 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-09 21:20:53,351 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-09 21:20:53,353 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-09 21:20:53,353 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-09 21:20:53,363 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-09 21:20:53,365 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-09 21:20:53,367 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-09 21:20:53,368 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-09 21:20:53,368 main DEBUG createAppenders(={Console})
2024-11-09 21:20:53,369 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-11-09 21:20:53,369 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-09 21:20:53,370 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-11-09 21:20:53,371 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-09 21:20:53,371 main DEBUG OutputStream closed
2024-11-09 21:20:53,371 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-09 21:20:53,372 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-09 21:20:53,372 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-11-09 21:20:53,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-09 21:20:53,459 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-09 21:20:53,460 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-09 21:20:53,461 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-09 21:20:53,461 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-09 21:20:53,462 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-09 21:20:53,462 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-09 21:20:53,462 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-09 21:20:53,463 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-09 21:20:53,463 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-09 21:20:53,463 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-09 21:20:53,463 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-09 21:20:53,464 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-09 21:20:53,464 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-09 21:20:53,464 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-09 21:20:53,465 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-09 21:20:53,465 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-09 21:20:53,466 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-09 21:20:53,468 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-09 21:20:53,468 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-11-09 21:20:53,468 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-09 21:20:53,469 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-11-09T21:20:53,711 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e
2024-11-09 21:20:53,713 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-09 21:20:53,714 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-09T21:20:53,722 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-11-09T21:20:53,756 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=25, ProcessCount=11, AvailableMemoryMB=19086
2024-11-09T21:20:53,759 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-09T21:20:53,773 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/cluster_bdeda732-fbb9-6548-402e-8714e08c557f, deleteOnExit=true
2024-11-09T21:20:53,773 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-09T21:20:53,774 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/test.cache.data in system properties and HBase conf
2024-11-09T21:20:53,775 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/hadoop.tmp.dir in system properties and HBase conf
2024-11-09T21:20:53,776 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/hadoop.log.dir in system properties and HBase conf
2024-11-09T21:20:53,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-09T21:20:53,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-09T21:20:53,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-09T21:20:53,867 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-09T21:20:53,956 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-09T21:20:53,960 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-09T21:20:53,961 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-09T21:20:53,961 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-09T21:20:53,962 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-09T21:20:53,962 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-09T21:20:53,963 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-09T21:20:53,963 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-09T21:20:53,963 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-09T21:20:53,964 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-09T21:20:53,964 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/nfs.dump.dir in system properties and HBase conf
2024-11-09T21:20:53,965 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/java.io.tmpdir in system properties and HBase conf
2024-11-09T21:20:53,965 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-09T21:20:53,965 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-09T21:20:53,966 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-09T21:20:54,431 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-09T21:20:54,959 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-09T21:20:55,030 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2448ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-09T21:20:55,099 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:20:55,166 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:20:55,188 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:20:55,188 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:20:55,189 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T21:20:55,201 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:20:55,204 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:20:55,205 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:20:55,379 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/java.io.tmpdir/jetty-localhost-46561-hadoop-hdfs-3_4_1-tests_jar-_-any-12339624991484158579/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-09T21:20:55,386 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:46561}
2024-11-09T21:20:55,386 INFO  [Time-limited test {}] server.Server(415): Started @2805ms
2024-11-09T21:20:55,410 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-09T21:20:56,881 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:20:56,888 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:20:56,889 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:20:56,889 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:20:56,890 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T21:20:56,890 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:20:56,891 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:20:56,992 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/java.io.tmpdir/jetty-localhost-46345-hadoop-hdfs-3_4_1-tests_jar-_-any-13014503692098012115/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:20:56,992 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:46345}
2024-11-09T21:20:56,993 INFO  [Time-limited test {}] server.Server(415): Started @4411ms
2024-11-09T21:20:57,046 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:20:57,162 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:20:57,169 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:20:57,174 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:20:57,174 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:20:57,174 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-09T21:20:57,176 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:20:57,177 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:20:57,309 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/java.io.tmpdir/jetty-localhost-35455-hadoop-hdfs-3_4_1-tests_jar-_-any-1596373244484210206/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:20:57,310 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:35455}
2024-11-09T21:20:57,310 INFO  [Time-limited test {}] server.Server(415): Started @4729ms
2024-11-09T21:20:57,312 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:20:58,281 WARN  [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/cluster_bdeda732-fbb9-6548-402e-8714e08c557f/data/data4/current/BP-651326839-172.17.0.2-1731187254518/current, will proceed with Du for space computation calculation, 
2024-11-09T21:20:58,281 WARN  [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/cluster_bdeda732-fbb9-6548-402e-8714e08c557f/data/data2/current/BP-651326839-172.17.0.2-1731187254518/current, will proceed with Du for space computation calculation, 
2024-11-09T21:20:58,281 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/cluster_bdeda732-fbb9-6548-402e-8714e08c557f/data/data3/current/BP-651326839-172.17.0.2-1731187254518/current, will proceed with Du for space computation calculation, 
2024-11-09T21:20:58,281 WARN  [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/cluster_bdeda732-fbb9-6548-402e-8714e08c557f/data/data1/current/BP-651326839-172.17.0.2-1731187254518/current, will proceed with Du for space computation calculation, 
2024-11-09T21:20:58,315 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:20:58,315 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:20:58,359 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd17e7f4754708fab with lease ID 0xc7ba2b382b49b23: Processing first storage report for DS-f286ad20-d3ff-4030-a970-8eecc87396e7 from datanode DatanodeRegistration(127.0.0.1:38913, datanodeUuid=f00a34af-c807-493f-9a4c-e4dc8ac42ce8, infoPort=35159, infoSecurePort=0, ipcPort=33165, storageInfo=lv=-57;cid=testClusterID;nsid=575340477;c=1731187254518)
2024-11-09T21:20:58,361 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd17e7f4754708fab with lease ID 0xc7ba2b382b49b23: from storage DS-f286ad20-d3ff-4030-a970-8eecc87396e7 node DatanodeRegistration(127.0.0.1:38913, datanodeUuid=f00a34af-c807-493f-9a4c-e4dc8ac42ce8, infoPort=35159, infoSecurePort=0, ipcPort=33165, storageInfo=lv=-57;cid=testClusterID;nsid=575340477;c=1731187254518), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-11-09T21:20:58,361 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x19c72baa5b299235 with lease ID 0xc7ba2b382b49b24: Processing first storage report for DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d from datanode DatanodeRegistration(127.0.0.1:40763, datanodeUuid=2e0e4656-3354-4639-8046-462951fa29c3, infoPort=34899, infoSecurePort=0, ipcPort=34069, storageInfo=lv=-57;cid=testClusterID;nsid=575340477;c=1731187254518)
2024-11-09T21:20:58,361 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x19c72baa5b299235 with lease ID 0xc7ba2b382b49b24: from storage DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d node DatanodeRegistration(127.0.0.1:40763, datanodeUuid=2e0e4656-3354-4639-8046-462951fa29c3, infoPort=34899, infoSecurePort=0, ipcPort=34069, storageInfo=lv=-57;cid=testClusterID;nsid=575340477;c=1731187254518), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:20:58,361 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd17e7f4754708fab with lease ID 0xc7ba2b382b49b23: Processing first storage report for DS-a6d45ff1-3bf6-4514-b406-f28078f9af41 from datanode DatanodeRegistration(127.0.0.1:38913, datanodeUuid=f00a34af-c807-493f-9a4c-e4dc8ac42ce8, infoPort=35159, infoSecurePort=0, ipcPort=33165, storageInfo=lv=-57;cid=testClusterID;nsid=575340477;c=1731187254518)
2024-11-09T21:20:58,362 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd17e7f4754708fab with lease ID 0xc7ba2b382b49b23: from storage DS-a6d45ff1-3bf6-4514-b406-f28078f9af41 node DatanodeRegistration(127.0.0.1:38913, datanodeUuid=f00a34af-c807-493f-9a4c-e4dc8ac42ce8, infoPort=35159, infoSecurePort=0, ipcPort=33165, storageInfo=lv=-57;cid=testClusterID;nsid=575340477;c=1731187254518), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:20:58,362 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x19c72baa5b299235 with lease ID 0xc7ba2b382b49b24: Processing first storage report for DS-b894f9fe-444a-47f7-ad23-e3f084046104 from datanode DatanodeRegistration(127.0.0.1:40763, datanodeUuid=2e0e4656-3354-4639-8046-462951fa29c3, infoPort=34899, infoSecurePort=0, ipcPort=34069, storageInfo=lv=-57;cid=testClusterID;nsid=575340477;c=1731187254518)
2024-11-09T21:20:58,362 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x19c72baa5b299235 with lease ID 0xc7ba2b382b49b24: from storage DS-b894f9fe-444a-47f7-ad23-e3f084046104 node DatanodeRegistration(127.0.0.1:40763, datanodeUuid=2e0e4656-3354-4639-8046-462951fa29c3, infoPort=34899, infoSecurePort=0, ipcPort=34069, storageInfo=lv=-57;cid=testClusterID;nsid=575340477;c=1731187254518), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:20:58,379 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e
2024-11-09T21:20:58,438 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/cluster_bdeda732-fbb9-6548-402e-8714e08c557f/zookeeper_0, clientPort=50470, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/cluster_bdeda732-fbb9-6548-402e-8714e08c557f/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/cluster_bdeda732-fbb9-6548-402e-8714e08c557f/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-09T21:20:58,446 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50470
2024-11-09T21:20:58,455 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:20:58,457 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:20:58,645 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741825_1001 (size=7)
2024-11-09T21:20:58,646 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741825_1001 (size=7)
2024-11-09T21:20:59,051 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8 with version=8
2024-11-09T21:20:59,051 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/hbase-staging
2024-11-09T21:20:59,134 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-09T21:20:59,371 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d86ff69d6719:0 server-side Connection retries=45
2024-11-09T21:20:59,379 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:20:59,380 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T21:20:59,385 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T21:20:59,386 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:20:59,386 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T21:20:59,513 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-09T21:20:59,567 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-09T21:20:59,575 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-09T21:20:59,578 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T21:20:59,600 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 22413 (auto-detected)
2024-11-09T21:20:59,601 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-09T21:20:59,618 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46577
2024-11-09T21:20:59,638 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46577 connecting to ZooKeeper ensemble=127.0.0.1:50470
2024-11-09T21:20:59,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:465770x0, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T21:20:59,760 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46577-0x10122c2c6fb0000 connected
2024-11-09T21:20:59,833 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:20:59,835 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:20:59,846 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:20:59,850 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8, hbase.cluster.distributed=false
2024-11-09T21:20:59,877 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T21:20:59,882 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46577
2024-11-09T21:20:59,882 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46577
2024-11-09T21:20:59,884 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46577
2024-11-09T21:20:59,885 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46577
2024-11-09T21:20:59,885 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46577
2024-11-09T21:20:59,983 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d86ff69d6719:0 server-side Connection retries=45
2024-11-09T21:20:59,984 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:20:59,985 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T21:20:59,985 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T21:20:59,985 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:20:59,985 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T21:20:59,988 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-09T21:20:59,991 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T21:20:59,992 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46221
2024-11-09T21:20:59,994 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46221 connecting to ZooKeeper ensemble=127.0.0.1:50470
2024-11-09T21:20:59,995 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:20:59,999 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:21:00,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:462210x0, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T21:21:00,032 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:462210x0, quorum=127.0.0.1:50470, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:21:00,032 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46221-0x10122c2c6fb0001 connected
2024-11-09T21:21:00,037 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-09T21:21:00,049 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-09T21:21:00,052 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-09T21:21:00,058 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T21:21:00,059 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46221
2024-11-09T21:21:00,060 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46221
2024-11-09T21:21:00,060 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46221
2024-11-09T21:21:00,060 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46221
2024-11-09T21:21:00,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46221
2024-11-09T21:21:00,075 DEBUG [M:0;d86ff69d6719:46577 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d86ff69d6719:46577
2024-11-09T21:21:00,076 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d86ff69d6719,46577,1731187259225
2024-11-09T21:21:00,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:21:00,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:21:00,092 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d86ff69d6719,46577,1731187259225
2024-11-09T21:21:00,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-09T21:21:00,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:21:00,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:21:00,115 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-09T21:21:00,116 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d86ff69d6719,46577,1731187259225 from backup master directory
2024-11-09T21:21:00,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d86ff69d6719,46577,1731187259225
2024-11-09T21:21:00,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:21:00,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:21:00,124 WARN  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T21:21:00,124 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d86ff69d6719,46577,1731187259225
2024-11-09T21:21:00,126 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-09T21:21:00,127 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-09T21:21:00,178 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/hbase.id] with ID: 8db3121c-3f36-438d-bc65-df8592b82358
2024-11-09T21:21:00,178 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/.tmp/hbase.id
2024-11-09T21:21:00,189 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741826_1002 (size=42)
2024-11-09T21:21:00,190 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741826_1002 (size=42)
2024-11-09T21:21:00,191 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/.tmp/hbase.id]:[hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/hbase.id]
2024-11-09T21:21:00,235 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:21:00,240 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-09T21:21:00,258 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms.
2024-11-09T21:21:00,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:21:00,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:21:00,288 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741827_1003 (size=196)
2024-11-09T21:21:00,288 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741827_1003 (size=196)
2024-11-09T21:21:00,307 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-09T21:21:00,310 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-09T21:21:00,316 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:21:00,346 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741828_1004 (size=1189)
2024-11-09T21:21:00,346 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741828_1004 (size=1189)
2024-11-09T21:21:00,361 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store
2024-11-09T21:21:00,380 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741829_1005 (size=34)
2024-11-09T21:21:00,380 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741829_1005 (size=34)
2024-11-09T21:21:00,385 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-09T21:21:00,388 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:21:00,390 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-09T21:21:00,390 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:21:00,390 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:21:00,392 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-09T21:21:00,392 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:21:00,392 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:21:00,394 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731187260390Disabling compacts and flushes for region at 1731187260390Disabling writes for close at 1731187260392 (+2 ms)Writing region close event to WAL at 1731187260392Closed at 1731187260392
2024-11-09T21:21:00,396 WARN  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/.initializing
2024-11-09T21:21:00,396 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/WALs/d86ff69d6719,46577,1731187259225
2024-11-09T21:21:00,417 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C46577%2C1731187259225, suffix=, logDir=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/WALs/d86ff69d6719,46577,1731187259225, archiveDir=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/oldWALs, maxLogs=10
2024-11-09T21:21:00,425 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C46577%2C1731187259225.1731187260422
2024-11-09T21:21:00,443 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/WALs/d86ff69d6719,46577,1731187259225/d86ff69d6719%2C46577%2C1731187259225.1731187260422
2024-11-09T21:21:00,449 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34899:34899),(127.0.0.1/127.0.0.1:35159:35159)]
2024-11-09T21:21:00,451 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:21:00,451 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:21:00,454 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:21:00,455 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:21:00,490 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:21:00,515 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-09T21:21:00,518 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:21:00,521 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:21:00,522 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:21:00,525 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-09T21:21:00,525 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:21:00,526 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:21:00,527 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:21:00,530 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-09T21:21:00,530 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:21:00,531 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:21:00,531 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:21:00,534 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-09T21:21:00,534 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:21:00,535 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:21:00,535 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:21:00,539 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:21:00,540 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:21:00,545 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:21:00,545 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:21:00,548 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-09T21:21:00,552 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:21:00,556 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:21:00,558 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=879408, jitterRate=0.11822567880153656}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-09T21:21:00,565 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731187260466Initializing all the Stores at 1731187260468 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187260469 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187260469Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187260470 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187260470Cleaning up temporary data from old regions at 1731187260545 (+75 ms)Region opened successfully at 1731187260565 (+20 ms)
2024-11-09T21:21:00,566 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-09T21:21:00,596 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3161f390, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0
2024-11-09T21:21:00,623 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-09T21:21:00,633 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-09T21:21:00,633 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-09T21:21:00,636 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-09T21:21:00,637 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-09T21:21:00,642 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-09T21:21:00,642 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-09T21:21:00,664 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-09T21:21:00,672 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-09T21:21:00,706 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-09T21:21:00,708 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-09T21:21:00,710 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-09T21:21:00,714 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-09T21:21:00,717 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-09T21:21:00,721 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-09T21:21:00,731 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-09T21:21:00,733 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-09T21:21:00,739 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-09T21:21:00,756 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-09T21:21:00,764 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-09T21:21:00,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-09T21:21:00,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-09T21:21:00,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:21:00,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:21:00,775 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d86ff69d6719,46577,1731187259225, sessionid=0x10122c2c6fb0000, setting cluster-up flag (Was=false)
2024-11-09T21:21:00,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:21:00,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:21:00,823 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-09T21:21:00,824 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d86ff69d6719,46577,1731187259225
2024-11-09T21:21:00,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:21:00,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:21:00,864 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-09T21:21:00,866 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d86ff69d6719,46577,1731187259225
2024-11-09T21:21:00,873 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-09T21:21:00,951 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-09T21:21:00,961 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-09T21:21:00,965 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer(746): ClusterId : 8db3121c-3f36-438d-bc65-df8592b82358
2024-11-09T21:21:00,967 DEBUG [RS:0;d86ff69d6719:46221 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-09T21:21:00,969 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-09T21:21:00,974 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d86ff69d6719,46577,1731187259225
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-09T21:21:00,983 DEBUG [RS:0;d86ff69d6719:46221 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-09T21:21:00,983 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:21:00,983 DEBUG [RS:0;d86ff69d6719:46221 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-09T21:21:00,983 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:21:00,983 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:21:00,983 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:21:00,984 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d86ff69d6719:0, corePoolSize=10, maxPoolSize=10
2024-11-09T21:21:00,984 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:21:00,984 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2
2024-11-09T21:21:00,984 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:21:00,987 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731187290987
2024-11-09T21:21:00,988 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-09T21:21:00,989 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-09T21:21:00,991 DEBUG [RS:0;d86ff69d6719:46221 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-09T21:21:00,991 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:21:00,992 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-09T21:21:00,992 DEBUG [RS:0;d86ff69d6719:46221 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@752f493a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0
2024-11-09T21:21:00,994 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-09T21:21:00,994 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-09T21:21:00,994 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-09T21:21:00,995 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-09T21:21:00,996 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:00,998 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:21:00,999 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-09T21:21:01,000 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-09T21:21:01,002 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-09T21:21:01,002 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-09T21:21:01,007 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-09T21:21:01,007 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-09T21:21:01,010 DEBUG [RS:0;d86ff69d6719:46221 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d86ff69d6719:46221
2024-11-09T21:21:01,011 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187261009,5,FailOnTimeoutGroup]
2024-11-09T21:21:01,013 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-09T21:21:01,013 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-09T21:21:01,013 DEBUG [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-09T21:21:01,015 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187261011,5,FailOnTimeoutGroup]
2024-11-09T21:21:01,015 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:01,015 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-09T21:21:01,016 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer(2659): reportForDuty to master=d86ff69d6719,46577,1731187259225 with port=46221, startcode=1731187259950
2024-11-09T21:21:01,017 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:01,017 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:01,017 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741831_1007 (size=1321)
2024-11-09T21:21:01,018 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741831_1007 (size=1321)
2024-11-09T21:21:01,020 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-09T21:21:01,020 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8
2024-11-09T21:21:01,032 DEBUG [RS:0;d86ff69d6719:46221 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-09T21:21:01,038 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741832_1008 (size=32)
2024-11-09T21:21:01,039 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741832_1008 (size=32)
2024-11-09T21:21:01,040 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:21:01,043 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-09T21:21:01,046 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-09T21:21:01,046 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:21:01,047 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:21:01,048 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-09T21:21:01,051 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-09T21:21:01,051 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:21:01,052 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:21:01,053 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-09T21:21:01,056 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-09T21:21:01,056 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:21:01,057 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:21:01,058 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-09T21:21:01,061 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-09T21:21:01,061 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:21:01,062 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:21:01,063 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-09T21:21:01,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740
2024-11-09T21:21:01,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740
2024-11-09T21:21:01,070 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-09T21:21:01,070 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-09T21:21:01,071 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-09T21:21:01,075 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-09T21:21:01,084 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:21:01,085 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=786426, jitterRate=-8.38935375213623E-6}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-09T21:21:01,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731187261041Initializing all the Stores at 1731187261042 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187261043 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187261043Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187261043Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187261043Cleaning up temporary data from old regions at 1731187261070 (+27 ms)Region opened successfully at 1731187261091 (+21 ms)
2024-11-09T21:21:01,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-09T21:21:01,091 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-09T21:21:01,092 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-09T21:21:01,092 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-09T21:21:01,092 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-09T21:21:01,093 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-09T21:21:01,093 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731187261091Disabling compacts and flushes for region at 1731187261091Disabling writes for close at 1731187261092 (+1 ms)Writing region close event to WAL at 1731187261093 (+1 ms)Closed at 1731187261093
2024-11-09T21:21:01,097 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:21:01,097 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-09T21:21:01,103 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-09T21:21:01,105 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38845, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-09T21:21:01,111 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46577 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d86ff69d6719,46221,1731187259950
2024-11-09T21:21:01,112 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-09T21:21:01,113 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46577 {}] master.ServerManager(517): Registering regionserver=d86ff69d6719,46221,1731187259950
2024-11-09T21:21:01,114 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-09T21:21:01,125 DEBUG [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8
2024-11-09T21:21:01,125 DEBUG [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34451
2024-11-09T21:21:01,125 DEBUG [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-09T21:21:01,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T21:21:01,148 DEBUG [RS:0;d86ff69d6719:46221 {}] zookeeper.ZKUtil(111): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d86ff69d6719,46221,1731187259950
2024-11-09T21:21:01,149 WARN  [RS:0;d86ff69d6719:46221 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T21:21:01,149 INFO  [RS:0;d86ff69d6719:46221 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:21:01,149 DEBUG [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950
2024-11-09T21:21:01,151 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d86ff69d6719,46221,1731187259950]
2024-11-09T21:21:01,176 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-09T21:21:01,189 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-09T21:21:01,194 INFO  [RS:0;d86ff69d6719:46221 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-09T21:21:01,194 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:01,195 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-09T21:21:01,200 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-09T21:21:01,201 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:01,201 DEBUG [RS:0;d86ff69d6719:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:21:01,201 DEBUG [RS:0;d86ff69d6719:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:21:01,201 DEBUG [RS:0;d86ff69d6719:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:21:01,202 DEBUG [RS:0;d86ff69d6719:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:21:01,202 DEBUG [RS:0;d86ff69d6719:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:21:01,202 DEBUG [RS:0;d86ff69d6719:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2
2024-11-09T21:21:01,202 DEBUG [RS:0;d86ff69d6719:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:21:01,202 DEBUG [RS:0;d86ff69d6719:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:21:01,202 DEBUG [RS:0;d86ff69d6719:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:21:01,202 DEBUG [RS:0;d86ff69d6719:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:21:01,203 DEBUG [RS:0;d86ff69d6719:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:21:01,203 DEBUG [RS:0;d86ff69d6719:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:21:01,203 DEBUG [RS:0;d86ff69d6719:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3
2024-11-09T21:21:01,203 DEBUG [RS:0;d86ff69d6719:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3
2024-11-09T21:21:01,204 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:01,204 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:01,204 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:01,204 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:01,204 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:01,204 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46221,1731187259950-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T21:21:01,221 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-09T21:21:01,223 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46221,1731187259950-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:01,223 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:01,223 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.Replication(171): d86ff69d6719,46221,1731187259950 started
2024-11-09T21:21:01,239 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:01,239 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer(1482): Serving as d86ff69d6719,46221,1731187259950, RpcServer on d86ff69d6719/172.17.0.2:46221, sessionid=0x10122c2c6fb0001
2024-11-09T21:21:01,240 DEBUG [RS:0;d86ff69d6719:46221 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-09T21:21:01,240 DEBUG [RS:0;d86ff69d6719:46221 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d86ff69d6719,46221,1731187259950
2024-11-09T21:21:01,240 DEBUG [RS:0;d86ff69d6719:46221 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,46221,1731187259950'
2024-11-09T21:21:01,240 DEBUG [RS:0;d86ff69d6719:46221 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-09T21:21:01,241 DEBUG [RS:0;d86ff69d6719:46221 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-09T21:21:01,242 DEBUG [RS:0;d86ff69d6719:46221 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-09T21:21:01,242 DEBUG [RS:0;d86ff69d6719:46221 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-09T21:21:01,242 DEBUG [RS:0;d86ff69d6719:46221 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d86ff69d6719,46221,1731187259950
2024-11-09T21:21:01,242 DEBUG [RS:0;d86ff69d6719:46221 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,46221,1731187259950'
2024-11-09T21:21:01,242 DEBUG [RS:0;d86ff69d6719:46221 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-09T21:21:01,243 DEBUG [RS:0;d86ff69d6719:46221 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-09T21:21:01,244 DEBUG [RS:0;d86ff69d6719:46221 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-09T21:21:01,244 INFO  [RS:0;d86ff69d6719:46221 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-09T21:21:01,244 INFO  [RS:0;d86ff69d6719:46221 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-09T21:21:01,265 WARN  [d86ff69d6719:46577 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-09T21:21:01,352 INFO  [RS:0;d86ff69d6719:46221 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C46221%2C1731187259950, suffix=, logDir=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950, archiveDir=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/oldWALs, maxLogs=32
2024-11-09T21:21:01,356 INFO  [RS:0;d86ff69d6719:46221 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C46221%2C1731187259950.1731187261355
2024-11-09T21:21:01,365 INFO  [RS:0;d86ff69d6719:46221 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187261355
2024-11-09T21:21:01,366 DEBUG [RS:0;d86ff69d6719:46221 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35159:35159),(127.0.0.1/127.0.0.1:34899:34899)]
2024-11-09T21:21:01,517 DEBUG [d86ff69d6719:46577 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-09T21:21:01,528 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d86ff69d6719,46221,1731187259950
2024-11-09T21:21:01,535 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d86ff69d6719,46221,1731187259950, state=OPENING
2024-11-09T21:21:01,643 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-09T21:21:01,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:21:01,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:21:01,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:21:01,667 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:21:01,670 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-09T21:21:01,674 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d86ff69d6719,46221,1731187259950}]
2024-11-09T21:21:01,845 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-09T21:21:01,849 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42779, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-09T21:21:01,860 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-09T21:21:01,860 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:21:01,864 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C46221%2C1731187259950.meta, suffix=.meta, logDir=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950, archiveDir=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/oldWALs, maxLogs=32
2024-11-09T21:21:01,866 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C46221%2C1731187259950.meta.1731187261866.meta
2024-11-09T21:21:01,874 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.meta.1731187261866.meta
2024-11-09T21:21:01,875 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34899:34899),(127.0.0.1/127.0.0.1:35159:35159)]
2024-11-09T21:21:01,876 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:21:01,878 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-09T21:21:01,880 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-09T21:21:01,884 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-09T21:21:01,888 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-09T21:21:01,888 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:21:01,889 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-09T21:21:01,889 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-09T21:21:01,891 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-09T21:21:01,893 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-09T21:21:01,893 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:21:01,894 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:21:01,894 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-09T21:21:01,896 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-09T21:21:01,896 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:21:01,897 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:21:01,897 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-09T21:21:01,899 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-09T21:21:01,899 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:21:01,899 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:21:01,900 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-09T21:21:01,901 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-09T21:21:01,901 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:21:01,902 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:21:01,902 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-09T21:21:01,904 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740
2024-11-09T21:21:01,906 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740
2024-11-09T21:21:01,909 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-09T21:21:01,909 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-09T21:21:01,910 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-09T21:21:01,912 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-09T21:21:01,914 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=778461, jitterRate=-0.010135695338249207}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-09T21:21:01,914 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-09T21:21:01,915 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731187261889Writing region info on filesystem at 1731187261889Initializing all the Stores at 1731187261891 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187261891Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187261891Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187261891Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187261891Cleaning up temporary data from old regions at 1731187261909 (+18 ms)Running coprocessor post-open hooks at 1731187261914 (+5 ms)Region opened successfully at 1731187261915 (+1 ms)
2024-11-09T21:21:01,922 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731187261838
2024-11-09T21:21:01,933 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-09T21:21:01,934 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-09T21:21:01,936 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,46221,1731187259950
2024-11-09T21:21:01,938 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d86ff69d6719,46221,1731187259950, state=OPEN
2024-11-09T21:21:01,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-09T21:21:01,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-09T21:21:01,987 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:21:01,987 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:21:01,987 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d86ff69d6719,46221,1731187259950
2024-11-09T21:21:01,994 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-09T21:21:01,995 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d86ff69d6719,46221,1731187259950 in 314 msec
2024-11-09T21:21:02,004 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-09T21:21:02,004 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 895 msec
2024-11-09T21:21:02,007 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:21:02,007 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-09T21:21:02,024 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-09T21:21:02,025 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,46221,1731187259950, seqNum=-1]
2024-11-09T21:21:02,042 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T21:21:02,043 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42045, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T21:21:02,064 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1530 sec
2024-11-09T21:21:02,064 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731187262064, completionTime=-1
2024-11-09T21:21:02,066 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-09T21:21:02,067 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-09T21:21:02,090 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-09T21:21:02,091 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731187322091
2024-11-09T21:21:02,091 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731187382091
2024-11-09T21:21:02,091 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec
2024-11-09T21:21:02,093 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46577,1731187259225-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:02,093 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46577,1731187259225-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:02,093 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46577,1731187259225-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:02,095 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d86ff69d6719:46577, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:02,095 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:02,096 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:02,102 DEBUG [master/d86ff69d6719:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-09T21:21:02,126 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.002sec
2024-11-09T21:21:02,127 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-09T21:21:02,128 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-09T21:21:02,129 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-09T21:21:02,130 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-09T21:21:02,130 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-09T21:21:02,131 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46577,1731187259225-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T21:21:02,147 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46577,1731187259225-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-09T21:21:02,157 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-09T21:21:02,158 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-09T21:21:02,158 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46577,1731187259225-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:21:02,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66cb686a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:21:02,180 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-09T21:21:02,180 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-09T21:21:02,185 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d86ff69d6719,46577,-1 for getting cluster id
2024-11-09T21:21:02,188 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-09T21:21:02,197 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8db3121c-3f36-438d-bc65-df8592b82358'
2024-11-09T21:21:02,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-09T21:21:02,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8db3121c-3f36-438d-bc65-df8592b82358"

2024-11-09T21:21:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59f6a6d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:21:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d86ff69d6719,46577,-1]
2024-11-09T21:21:02,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-09T21:21:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:21:02,205 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-09T21:21:02,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32fb6022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:21:02,209 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-09T21:21:02,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,46221,1731187259950, seqNum=-1]
2024-11-09T21:21:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T21:21:02,220 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T21:21:02,240 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d86ff69d6719,46577,1731187259225
2024-11-09T21:21:02,240 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:21:02,246 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-09T21:21:02,250 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-09T21:21:02,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d86ff69d6719,46577,1731187259225
2024-11-09T21:21:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@36ee107f
2024-11-09T21:21:02,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-09T21:21:02,260 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-09T21:21:02,262 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46577 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-09T21:21:02,262 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46577 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-09T21:21:02,265 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46577 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-09T21:21:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46577 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-11-09T21:21:02,276 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-09T21:21:02,278 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46577 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-11-09T21:21:02,278 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:21:02,281 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-09T21:21:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-09T21:21:02,338 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741835_1011 (size=389)
2024-11-09T21:21:02,339 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741835_1011 (size=389)
2024-11-09T21:21:02,342 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d0baaae191fb009293911b746e9a4b1d, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8
2024-11-09T21:21:02,352 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741836_1012 (size=72)
2024-11-09T21:21:02,353 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741836_1012 (size=72)
2024-11-09T21:21:02,354 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:21:02,354 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing d0baaae191fb009293911b746e9a4b1d, disabling compactions & flushes
2024-11-09T21:21:02,354 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.
2024-11-09T21:21:02,354 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.
2024-11-09T21:21:02,354 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d. after waiting 0 ms
2024-11-09T21:21:02,354 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.
2024-11-09T21:21:02,354 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.
2024-11-09T21:21:02,354 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for d0baaae191fb009293911b746e9a4b1d:
Waiting for close lock at 1731187262354Disabling compacts and flushes for region at 1731187262354Disabling writes for close at 1731187262354Writing region close event to WAL at 1731187262354Closed at 1731187262354
2024-11-09T21:21:02,357 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-09T21:21:02,362 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731187262357"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731187262357"}]},"ts":"1731187262357"}
2024-11-09T21:21:02,368 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-09T21:21:02,370 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-09T21:21:02,372 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187262370"}]},"ts":"1731187262370"}
2024-11-09T21:21:02,377 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-11-09T21:21:02,379 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=d0baaae191fb009293911b746e9a4b1d, ASSIGN}]
2024-11-09T21:21:02,382 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=d0baaae191fb009293911b746e9a4b1d, ASSIGN
2024-11-09T21:21:02,384 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=d0baaae191fb009293911b746e9a4b1d, ASSIGN; state=OFFLINE, location=d86ff69d6719,46221,1731187259950; forceNewPlan=false, retain=false
2024-11-09T21:21:02,535 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d0baaae191fb009293911b746e9a4b1d, regionState=OPENING, regionLocation=d86ff69d6719,46221,1731187259950
2024-11-09T21:21:02,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=d0baaae191fb009293911b746e9a4b1d, ASSIGN because future has completed
2024-11-09T21:21:02,540 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0baaae191fb009293911b746e9a4b1d, server=d86ff69d6719,46221,1731187259950}]
2024-11-09T21:21:02,701 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.
2024-11-09T21:21:02,701 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d0baaae191fb009293911b746e9a4b1d, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:21:02,701 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling d0baaae191fb009293911b746e9a4b1d
2024-11-09T21:21:02,702 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:21:02,702 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d0baaae191fb009293911b746e9a4b1d
2024-11-09T21:21:02,702 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d0baaae191fb009293911b746e9a4b1d
2024-11-09T21:21:02,705 INFO  [StoreOpener-d0baaae191fb009293911b746e9a4b1d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d0baaae191fb009293911b746e9a4b1d 
2024-11-09T21:21:02,708 INFO  [StoreOpener-d0baaae191fb009293911b746e9a4b1d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0baaae191fb009293911b746e9a4b1d columnFamilyName info
2024-11-09T21:21:02,708 DEBUG [StoreOpener-d0baaae191fb009293911b746e9a4b1d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:21:02,710 INFO  [StoreOpener-d0baaae191fb009293911b746e9a4b1d-1 {}] regionserver.HStore(327): Store=d0baaae191fb009293911b746e9a4b1d/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:21:02,710 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d0baaae191fb009293911b746e9a4b1d
2024-11-09T21:21:02,712 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d
2024-11-09T21:21:02,713 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d
2024-11-09T21:21:02,714 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d0baaae191fb009293911b746e9a4b1d
2024-11-09T21:21:02,714 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d0baaae191fb009293911b746e9a4b1d
2024-11-09T21:21:02,718 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d0baaae191fb009293911b746e9a4b1d
2024-11-09T21:21:02,722 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:21:02,723 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d0baaae191fb009293911b746e9a4b1d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=726901, jitterRate=-0.07569839060306549}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-09T21:21:02,724 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d0baaae191fb009293911b746e9a4b1d
2024-11-09T21:21:02,725 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d0baaae191fb009293911b746e9a4b1d:
Running coprocessor pre-open hook at 1731187262702Writing region info on filesystem at 1731187262702Initializing all the Stores at 1731187262704 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187262704Cleaning up temporary data from old regions at 1731187262714 (+10 ms)Running coprocessor post-open hooks at 1731187262724 (+10 ms)Region opened successfully at 1731187262725 (+1 ms)
2024-11-09T21:21:02,728 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d., pid=6, masterSystemTime=1731187262694
2024-11-09T21:21:02,732 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.
2024-11-09T21:21:02,732 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.
2024-11-09T21:21:02,734 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d0baaae191fb009293911b746e9a4b1d, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,46221,1731187259950
2024-11-09T21:21:02,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0baaae191fb009293911b746e9a4b1d, server=d86ff69d6719,46221,1731187259950 because future has completed
2024-11-09T21:21:02,746 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-09T21:21:02,746 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d0baaae191fb009293911b746e9a4b1d, server=d86ff69d6719,46221,1731187259950 in 201 msec
2024-11-09T21:21:02,752 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-09T21:21:02,752 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=d0baaae191fb009293911b746e9a4b1d, ASSIGN in 367 msec
2024-11-09T21:21:02,754 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-09T21:21:02,755 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187262754"}]},"ts":"1731187262754"}
2024-11-09T21:21:02,759 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-11-09T21:21:02,761 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-09T21:21:02,767 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 493 msec
2024-11-09T21:21:07,397 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-09T21:21:07,453 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-09T21:21:07,454 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-11-09T21:21:09,564 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-09T21:21:09,564 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-09T21:21:09,566 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-09T21:21:09,566 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-09T21:21:09,567 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-09T21:21:09,567 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-09T21:21:09,567 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-09T21:21:09,567 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-09T21:21:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-09T21:21:12,364 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-11-09T21:21:12,366 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-11-09T21:21:12,372 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-11-09T21:21:12,373 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.
2024-11-09T21:21:12,374 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C46221%2C1731187259950.1731187272373
2024-11-09T21:21:12,382 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:12,382 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:12,382 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:12,383 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:12,383 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:12,383 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187261355 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187272373
2024-11-09T21:21:12,385 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35159:35159),(127.0.0.1/127.0.0.1:34899:34899)]
2024-11-09T21:21:12,385 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187261355 is not closed yet, will try archiving it next time
2024-11-09T21:21:12,386 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741833_1009 (size=451)
2024-11-09T21:21:12,387 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741833_1009 (size=451)
2024-11-09T21:21:12,388 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187261355 to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/oldWALs/d86ff69d6719%2C46221%2C1731187259950.1731187261355
2024-11-09T21:21:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d., hostname=d86ff69d6719,46221,1731187259950, seqNum=2]
2024-11-09T21:21:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on d0baaae191fb009293911b746e9a4b1d
2024-11-09T21:21:24,432 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0baaae191fb009293911b746e9a4b1d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-09T21:21:24,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/ccbc649cb13e4c22b9a1906128d11bd2 is 1080, key is row0001/info:/1731187272397/Put/seqid=0
2024-11-09T21:21:24,501 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741838_1014 (size=12509)
2024-11-09T21:21:24,501 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741838_1014 (size=12509)
2024-11-09T21:21:24,502 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/ccbc649cb13e4c22b9a1906128d11bd2
2024-11-09T21:21:24,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/ccbc649cb13e4c22b9a1906128d11bd2 as hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/ccbc649cb13e4c22b9a1906128d11bd2
2024-11-09T21:21:24,563 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/ccbc649cb13e4c22b9a1906128d11bd2, entries=7, sequenceid=11, filesize=12.2 K
2024-11-09T21:21:24,571 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for d0baaae191fb009293911b746e9a4b1d in 137ms, sequenceid=11, compaction requested=false
2024-11-09T21:21:24,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0baaae191fb009293911b746e9a4b1d:

2024-11-09T21:21:28,377 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T21:21:32,495 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C46221%2C1731187259950.1731187292494
2024-11-09T21:21:32,703 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:21:32,704 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:32,704 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:32,704 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:32,704 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:32,704 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:32,705 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187272373 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187292494
2024-11-09T21:21:32,705 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35159:35159),(127.0.0.1/127.0.0.1:34899:34899)]
2024-11-09T21:21:32,706 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187272373 is not closed yet, will try archiving it next time
2024-11-09T21:21:32,707 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741837_1013 (size=12399)
2024-11-09T21:21:32,708 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741837_1013 (size=12399)
2024-11-09T21:21:32,909 INFO  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:21:35,115 INFO  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:21:37,319 INFO  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:21:39,523 INFO  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:21:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on d0baaae191fb009293911b746e9a4b1d
2024-11-09T21:21:39,523 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0baaae191fb009293911b746e9a4b1d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-09T21:21:39,726 INFO  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:21:39,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/eb20869df0584adea815473d2ad57025 is 1080, key is row0008/info:/1731187286431/Put/seqid=0
2024-11-09T21:21:39,745 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741840_1016 (size=12509)
2024-11-09T21:21:39,745 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741840_1016 (size=12509)
2024-11-09T21:21:39,747 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/eb20869df0584adea815473d2ad57025
2024-11-09T21:21:39,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/eb20869df0584adea815473d2ad57025 as hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/eb20869df0584adea815473d2ad57025
2024-11-09T21:21:39,766 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/eb20869df0584adea815473d2ad57025, entries=7, sequenceid=21, filesize=12.2 K
2024-11-09T21:21:39,968 INFO  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:21:39,969 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for d0baaae191fb009293911b746e9a4b1d in 445ms, sequenceid=21, compaction requested=false
2024-11-09T21:21:39,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0baaae191fb009293911b746e9a4b1d:

2024-11-09T21:21:39,969 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-11-09T21:21:39,969 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:21:39,970 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/ccbc649cb13e4c22b9a1906128d11bd2 because midkey is the same as first or last row
2024-11-09T21:21:41,728 INFO  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:21:42,962 INFO  [master/d86ff69d6719:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-09T21:21:42,962 INFO  [master/d86ff69d6719:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-09T21:21:43,938 INFO  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1368): Slow sync cost: 203 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:21:43,944 WARN  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:21:43,945 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d86ff69d6719%2C46221%2C1731187259950:(num 1731187292494) roll requested
2024-11-09T21:21:43,945 INFO  [regionserver/d86ff69d6719:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C46221%2C1731187259950.1731187303945
2024-11-09T21:21:44,159 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 211 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:21:44,159 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:44,159 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:44,159 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:44,160 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:44,160 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:44,160 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187292494 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187303945
2024-11-09T21:21:44,161 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34899:34899),(127.0.0.1/127.0.0.1:35159:35159)]
2024-11-09T21:21:44,161 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187292494 is not closed yet, will try archiving it next time
2024-11-09T21:21:44,162 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187272373 to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/oldWALs/d86ff69d6719%2C46221%2C1731187259950.1731187272373
2024-11-09T21:21:44,163 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741839_1015 (size=7739)
2024-11-09T21:21:44,163 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741839_1015 (size=7739)
2024-11-09T21:21:46,143 INFO  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK], DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK]]
2024-11-09T21:21:47,702 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region d0baaae191fb009293911b746e9a4b1d, had cached 0 bytes from a total of 25018
2024-11-09T21:21:48,349 INFO  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK], DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK]]
2024-11-09T21:21:50,554 INFO  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK], DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK]]
2024-11-09T21:21:52,763 INFO  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK], DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK]]
2024-11-09T21:21:54,768 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-09T21:21:54,770 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C46221%2C1731187259950.1731187314769
2024-11-09T21:21:58,377 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T21:21:59,786 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5012 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK], DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK]]
2024-11-09T21:21:59,789 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5012 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK], DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK]]
2024-11-09T21:21:59,789 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d86ff69d6719%2C46221%2C1731187259950:(num 1731187314769) roll requested
2024-11-09T21:21:59,789 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:59,789 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:59,790 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:59,790 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:59,790 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:21:59,790 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187303945 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187314769
2024-11-09T21:21:59,791 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35159:35159),(127.0.0.1/127.0.0.1:34899:34899)]
2024-11-09T21:21:59,791 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187303945 is not closed yet, will try archiving it next time
2024-11-09T21:21:59,792 INFO  [regionserver/d86ff69d6719:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C46221%2C1731187259950.1731187319792
2024-11-09T21:21:59,793 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741841_1017 (size=4753)
2024-11-09T21:21:59,794 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741841_1017 (size=4753)
2024-11-09T21:22:04,795 INFO  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:22:04,795 WARN  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:22:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on d0baaae191fb009293911b746e9a4b1d
2024-11-09T21:22:04,796 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0baaae191fb009293911b746e9a4b1d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-09T21:22:04,803 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:22:04,803 WARN  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:22:06,797 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-09T21:22:09,800 INFO  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5002 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:22:09,800 WARN  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5002 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:22:09,801 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:09,801 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:09,802 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:09,802 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:09,803 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:09,804 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187314769 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187319792
2024-11-09T21:22:09,806 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35159:35159),(127.0.0.1/127.0.0.1:34899:34899)]
2024-11-09T21:22:09,806 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187314769 is not closed yet, will try archiving it next time
2024-11-09T21:22:09,806 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d86ff69d6719%2C46221%2C1731187259950:(num 1731187319792) roll requested
2024-11-09T21:22:09,807 INFO  [regionserver/d86ff69d6719:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C46221%2C1731187259950.1731187329807
2024-11-09T21:22:09,808 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741842_1018 (size=1569)
2024-11-09T21:22:09,808 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741842_1018 (size=1569)
2024-11-09T21:22:09,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/e17c1c5c81ec4143a1107fb408d29c80 is 1080, key is row0015/info:/1731187301525/Put/seqid=0
2024-11-09T21:22:09,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741844_1020 (size=12509)
2024-11-09T21:22:09,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741844_1020 (size=12509)
2024-11-09T21:22:09,821 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/e17c1c5c81ec4143a1107fb408d29c80
2024-11-09T21:22:09,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/e17c1c5c81ec4143a1107fb408d29c80 as hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/e17c1c5c81ec4143a1107fb408d29c80
2024-11-09T21:22:09,841 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/e17c1c5c81ec4143a1107fb408d29c80, entries=7, sequenceid=31, filesize=12.2 K
2024-11-09T21:22:14,815 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:22:14,815 WARN  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:22:14,842 INFO  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:22:14,842 WARN  [FSHLog-0-hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8-prefix:d86ff69d6719,46221,1731187259950 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38913,DS-f286ad20-d3ff-4030-a970-8eecc87396e7,DISK], DatanodeInfoWithStorage[127.0.0.1:40763,DS-f52d99e9-44fa-4a33-b907-ed1b7cdd110d,DISK]]
2024-11-09T21:22:14,842 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for d0baaae191fb009293911b746e9a4b1d in 10046ms, sequenceid=31, compaction requested=true
2024-11-09T21:22:14,843 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:14,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0baaae191fb009293911b746e9a4b1d:

2024-11-09T21:22:14,843 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:14,843 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:14,843 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-11-09T21:22:14,843 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:22:14,843 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:14,843 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:14,843 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/ccbc649cb13e4c22b9a1906128d11bd2 because midkey is the same as first or last row
2024-11-09T21:22:14,843 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187319792 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187329807
2024-11-09T21:22:14,844 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35159:35159),(127.0.0.1/127.0.0.1:34899:34899)]
2024-11-09T21:22:14,844 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187319792 is not closed yet, will try archiving it next time
2024-11-09T21:22:14,845 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d86ff69d6719%2C46221%2C1731187259950:(num 1731187334845) roll requested
2024-11-09T21:22:14,845 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187292494 to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/oldWALs/d86ff69d6719%2C46221%2C1731187259950.1731187292494
2024-11-09T21:22:14,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0baaae191fb009293911b746e9a4b1d:info, priority=-2147483648, current under compaction store size is 1
2024-11-09T21:22:14,845 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C46221%2C1731187259950.1731187334845
2024-11-09T21:22:14,846 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741843_1019 (size=438)
2024-11-09T21:22:14,848 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187303945 to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/oldWALs/d86ff69d6719%2C46221%2C1731187259950.1731187303945
2024-11-09T21:22:14,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:22:14,849 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-09T21:22:14,849 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741843_1019 (size=438)
2024-11-09T21:22:14,851 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187314769 to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/oldWALs/d86ff69d6719%2C46221%2C1731187259950.1731187314769
2024-11-09T21:22:14,853 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187319792 to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/oldWALs/d86ff69d6719%2C46221%2C1731187259950.1731187319792
2024-11-09T21:22:14,853 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-09T21:22:14,855 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.HStore(1541): d0baaae191fb009293911b746e9a4b1d/info is initiating minor compaction (all files)
2024-11-09T21:22:14,855 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:14,856 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:14,856 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:14,856 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:14,856 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:14,856 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187329807 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187334845
2024-11-09T21:22:14,856 INFO  [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0baaae191fb009293911b746e9a4b1d/info in TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.
2024-11-09T21:22:14,857 INFO  [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/ccbc649cb13e4c22b9a1906128d11bd2, hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/eb20869df0584adea815473d2ad57025, hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/e17c1c5c81ec4143a1107fb408d29c80] into tmpdir=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp, totalSize=36.6 K
2024-11-09T21:22:14,857 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35159:35159),(127.0.0.1/127.0.0.1:34899:34899)]
2024-11-09T21:22:14,857 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187329807 is not closed yet, will try archiving it next time
2024-11-09T21:22:14,857 INFO  [regionserver/d86ff69d6719:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C46221%2C1731187259950.1731187334857
2024-11-09T21:22:14,858 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741845_1021 (size=93)
2024-11-09T21:22:14,859 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccbc649cb13e4c22b9a1906128d11bd2, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731187272397
2024-11-09T21:22:14,859 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741845_1021 (size=93)
2024-11-09T21:22:14,859 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187329807 to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/oldWALs/d86ff69d6719%2C46221%2C1731187259950.1731187329807
2024-11-09T21:22:14,862 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb20869df0584adea815473d2ad57025, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1731187286431
2024-11-09T21:22:14,863 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting e17c1c5c81ec4143a1107fb408d29c80, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731187301525
2024-11-09T21:22:14,869 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:14,869 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:14,869 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:14,869 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:14,869 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:14,870 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187334845 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187334857
2024-11-09T21:22:14,871 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34899:34899),(127.0.0.1/127.0.0.1:35159:35159)]
2024-11-09T21:22:14,871 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/WALs/d86ff69d6719,46221,1731187259950/d86ff69d6719%2C46221%2C1731187259950.1731187334845 is not closed yet, will try archiving it next time
2024-11-09T21:22:14,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741846_1022 (size=1258)
2024-11-09T21:22:14,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741846_1022 (size=1258)
2024-11-09T21:22:14,891 INFO  [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0baaae191fb009293911b746e9a4b1d#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-09T21:22:14,892 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/18b825554ef545b3947f61bfce841c9e is 1080, key is row0001/info:/1731187272397/Put/seqid=0
2024-11-09T21:22:14,898 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741848_1024 (size=27710)
2024-11-09T21:22:14,898 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741848_1024 (size=27710)
2024-11-09T21:22:14,909 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/18b825554ef545b3947f61bfce841c9e as hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/18b825554ef545b3947f61bfce841c9e
2024-11-09T21:22:14,926 INFO  [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0baaae191fb009293911b746e9a4b1d/info of d0baaae191fb009293911b746e9a4b1d into 18b825554ef545b3947f61bfce841c9e(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-09T21:22:14,926 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0baaae191fb009293911b746e9a4b1d:

2024-11-09T21:22:14,929 INFO  [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d., storeName=d0baaae191fb009293911b746e9a4b1d/info, priority=13, startTime=1731187334845; duration=0sec
2024-11-09T21:22:14,929 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-09T21:22:14,929 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:22:14,929 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/18b825554ef545b3947f61bfce841c9e because midkey is the same as first or last row
2024-11-09T21:22:14,929 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-09T21:22:14,929 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:22:14,929 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/18b825554ef545b3947f61bfce841c9e because midkey is the same as first or last row
2024-11-09T21:22:14,930 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-09T21:22:14,930 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:22:14,930 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/18b825554ef545b3947f61bfce841c9e because midkey is the same as first or last row
2024-11-09T21:22:14,930 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:22:14,930 DEBUG [RS:0;d86ff69d6719:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0baaae191fb009293911b746e9a4b1d:info
2024-11-09T21:22:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on d0baaae191fb009293911b746e9a4b1d
2024-11-09T21:22:26,905 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0baaae191fb009293911b746e9a4b1d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-09T21:22:26,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/2a39c32e0d734d629bd946c1c28c6c10 is 1080, key is row0022/info:/1731187334859/Put/seqid=0
2024-11-09T21:22:26,921 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741849_1025 (size=12509)
2024-11-09T21:22:26,922 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741849_1025 (size=12509)
2024-11-09T21:22:26,923 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/2a39c32e0d734d629bd946c1c28c6c10
2024-11-09T21:22:26,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/2a39c32e0d734d629bd946c1c28c6c10 as hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/2a39c32e0d734d629bd946c1c28c6c10
2024-11-09T21:22:26,943 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/2a39c32e0d734d629bd946c1c28c6c10, entries=7, sequenceid=42, filesize=12.2 K
2024-11-09T21:22:26,944 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for d0baaae191fb009293911b746e9a4b1d in 39ms, sequenceid=42, compaction requested=false
2024-11-09T21:22:26,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0baaae191fb009293911b746e9a4b1d:

2024-11-09T21:22:26,945 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-11-09T21:22:26,945 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:22:26,945 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/18b825554ef545b3947f61bfce841c9e because midkey is the same as first or last row
2024-11-09T21:22:28,377 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T21:22:32,702 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region d0baaae191fb009293911b746e9a4b1d, had cached 0 bytes from a total of 40219
2024-11-09T21:22:34,927 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-09T21:22:34,929 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-09T21:22:34,929 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:22:34,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:22:34,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:22:34,940 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-09T21:22:34,941 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-09T21:22:34,941 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=296299596, stopped=false
2024-11-09T21:22:34,941 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d86ff69d6719,46577,1731187259225
2024-11-09T21:22:34,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-09T21:22:34,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-09T21:22:34,985 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-09T21:22:34,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:34,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:34,985 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-09T21:22:34,985 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:22:34,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:22:34,986 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:22:34,986 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:22:34,986 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd86ff69d6719,46221,1731187259950' *****
2024-11-09T21:22:34,986 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-09T21:22:34,987 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-09T21:22:34,987 INFO  [RS:0;d86ff69d6719:46221 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-09T21:22:34,987 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-09T21:22:34,987 INFO  [RS:0;d86ff69d6719:46221 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-09T21:22:34,987 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer(3091): Received CLOSE for d0baaae191fb009293911b746e9a4b1d
2024-11-09T21:22:34,988 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer(959): stopping server d86ff69d6719,46221,1731187259950
2024-11-09T21:22:34,988 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-09T21:22:34,988 INFO  [RS:0;d86ff69d6719:46221 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d86ff69d6719:46221.
2024-11-09T21:22:34,988 DEBUG [RS:0;d86ff69d6719:46221 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:22:34,988 DEBUG [RS:0;d86ff69d6719:46221 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:22:34,988 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d0baaae191fb009293911b746e9a4b1d, disabling compactions & flushes
2024-11-09T21:22:34,989 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-09T21:22:34,989 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.
2024-11-09T21:22:34,989 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-09T21:22:34,989 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.
2024-11-09T21:22:34,989 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-09T21:22:34,989 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d. after waiting 0 ms
2024-11-09T21:22:34,989 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-09T21:22:34,989 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.
2024-11-09T21:22:34,989 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d0baaae191fb009293911b746e9a4b1d 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-09T21:22:34,989 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-09T21:22:34,989 DEBUG [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, d0baaae191fb009293911b746e9a4b1d=TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.}
2024-11-09T21:22:34,989 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-09T21:22:34,990 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-09T21:22:34,990 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-09T21:22:34,990 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-09T21:22:34,990 DEBUG [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d0baaae191fb009293911b746e9a4b1d
2024-11-09T21:22:34,990 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-09T21:22:34,990 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-11-09T21:22:34,997 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/7631ae0701e240d28c6e684243028eae is 1080, key is row0029/info:/1731187348907/Put/seqid=0
2024-11-09T21:22:35,003 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741850_1026 (size=8193)
2024-11-09T21:22:35,003 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741850_1026 (size=8193)
2024-11-09T21:22:35,004 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/7631ae0701e240d28c6e684243028eae
2024-11-09T21:22:35,012 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/.tmp/info/7631ae0701e240d28c6e684243028eae as hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/7631ae0701e240d28c6e684243028eae
2024-11-09T21:22:35,017 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/.tmp/info/980554145b354a79a8f0e8345a8faa2e is 195, key is TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d./info:regioninfo/1731187262733/Put/seqid=0
2024-11-09T21:22:35,021 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/7631ae0701e240d28c6e684243028eae, entries=3, sequenceid=48, filesize=8.0 K
2024-11-09T21:22:35,022 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for d0baaae191fb009293911b746e9a4b1d in 33ms, sequenceid=48, compaction requested=true
2024-11-09T21:22:35,023 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741851_1027 (size=7016)
2024-11-09T21:22:35,023 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/ccbc649cb13e4c22b9a1906128d11bd2, hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/eb20869df0584adea815473d2ad57025, hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/e17c1c5c81ec4143a1107fb408d29c80] to archive
2024-11-09T21:22:35,023 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741851_1027 (size=7016)
2024-11-09T21:22:35,023 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/.tmp/info/980554145b354a79a8f0e8345a8faa2e
2024-11-09T21:22:35,026 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-09T21:22:35,029 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/ccbc649cb13e4c22b9a1906128d11bd2 to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/archive/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/ccbc649cb13e4c22b9a1906128d11bd2
2024-11-09T21:22:35,031 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/eb20869df0584adea815473d2ad57025 to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/archive/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/eb20869df0584adea815473d2ad57025
2024-11-09T21:22:35,033 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/e17c1c5c81ec4143a1107fb408d29c80 to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/archive/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/info/e17c1c5c81ec4143a1107fb408d29c80
2024-11-09T21:22:35,043 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d86ff69d6719:46577 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-09T21:22:35,047 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [ccbc649cb13e4c22b9a1906128d11bd2=12509, eb20869df0584adea815473d2ad57025=12509, e17c1c5c81ec4143a1107fb408d29c80=12509]
2024-11-09T21:22:35,047 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/.tmp/ns/b8afe91895e5467d8752522f63d243e3 is 43, key is default/ns:d/1731187262047/Put/seqid=0
2024-11-09T21:22:35,055 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/default/TestLogRolling-testSlowSyncLogRolling/d0baaae191fb009293911b746e9a4b1d/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-11-09T21:22:35,056 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741852_1028 (size=5153)
2024-11-09T21:22:35,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741852_1028 (size=5153)
2024-11-09T21:22:35,057 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.
2024-11-09T21:22:35,058 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/.tmp/ns/b8afe91895e5467d8752522f63d243e3
2024-11-09T21:22:35,058 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d0baaae191fb009293911b746e9a4b1d:
Waiting for close lock at 1731187354988Running coprocessor pre-close hooks at 1731187354988Disabling compacts and flushes for region at 1731187354988Disabling writes for close at 1731187354989 (+1 ms)Obtaining lock to block concurrent updates at 1731187354989Preparing flush snapshotting stores in d0baaae191fb009293911b746e9a4b1d at 1731187354989Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1731187354990 (+1 ms)Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d. at 1731187354991 (+1 ms)Flushing d0baaae191fb009293911b746e9a4b1d/info: creating writer at 1731187354991Flushing d0baaae191fb009293911b746e9a4b1d/info: appending metadata at 1731187354996 (+5 ms)Flushing d0baaae191fb009293911b746e9a4b1d/info: closing flushed file at 1731187354996Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18696afe: reopening flushed file at 1731187355011 (+15 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for d0baaae191fb009293911b746e9a4b1d in 33ms, sequenceid=48, compaction requested=true at 1731187355022 (+11 ms)Writing region close event to WAL at 1731187355048 (+26 ms)Running coprocessor post-close hooks at 1731187355056 (+8 ms)Closed at 1731187355057 (+1 ms)
2024-11-09T21:22:35,058 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1731187262262.d0baaae191fb009293911b746e9a4b1d.
2024-11-09T21:22:35,080 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/.tmp/table/3f0cddaeb0154b63bd9f363a0cad8a02 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1731187262754/Put/seqid=0
2024-11-09T21:22:35,085 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741853_1029 (size=5396)
2024-11-09T21:22:35,085 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741853_1029 (size=5396)
2024-11-09T21:22:35,086 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/.tmp/table/3f0cddaeb0154b63bd9f363a0cad8a02
2024-11-09T21:22:35,093 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/.tmp/info/980554145b354a79a8f0e8345a8faa2e as hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/info/980554145b354a79a8f0e8345a8faa2e
2024-11-09T21:22:35,103 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/info/980554145b354a79a8f0e8345a8faa2e, entries=10, sequenceid=11, filesize=6.9 K
2024-11-09T21:22:35,104 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/.tmp/ns/b8afe91895e5467d8752522f63d243e3 as hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/ns/b8afe91895e5467d8752522f63d243e3
2024-11-09T21:22:35,113 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/ns/b8afe91895e5467d8752522f63d243e3, entries=2, sequenceid=11, filesize=5.0 K
2024-11-09T21:22:35,115 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/.tmp/table/3f0cddaeb0154b63bd9f363a0cad8a02 as hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/table/3f0cddaeb0154b63bd9f363a0cad8a02
2024-11-09T21:22:35,123 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/table/3f0cddaeb0154b63bd9f363a0cad8a02, entries=2, sequenceid=11, filesize=5.3 K
2024-11-09T21:22:35,124 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 134ms, sequenceid=11, compaction requested=false
2024-11-09T21:22:35,130 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-09T21:22:35,131 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-09T21:22:35,131 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-09T21:22:35,132 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731187354989Running coprocessor pre-close hooks at 1731187354989Disabling compacts and flushes for region at 1731187354989Disabling writes for close at 1731187354990 (+1 ms)Obtaining lock to block concurrent updates at 1731187354990Preparing flush snapshotting stores in 1588230740 at 1731187354990Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1731187354991 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731187354992 (+1 ms)Flushing 1588230740/info: creating writer at 1731187354992Flushing 1588230740/info: appending metadata at 1731187355016 (+24 ms)Flushing 1588230740/info: closing flushed file at 1731187355016Flushing 1588230740/ns: creating writer at 1731187355032 (+16 ms)Flushing 1588230740/ns: appending metadata at 1731187355047 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731187355047Flushing 1588230740/table: creating writer at 1731187355065 (+18 ms)Flushing 1588230740/table: appending metadata at 1731187355079 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731187355079Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24eb65fe: reopening flushed file at 1731187355092 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@687c83a7: reopening flushed file at 1731187355103 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2042e636: reopening flushed file at 1731187355113 (+10 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 134ms, sequenceid=11, compaction requested=false at 1731187355124 (+11 ms)Writing region close event to WAL at 1731187355126 (+2 ms)Running coprocessor post-close hooks at 1731187355131 (+5 ms)Closed at 1731187355131
2024-11-09T21:22:35,132 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-09T21:22:35,190 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer(976): stopping server d86ff69d6719,46221,1731187259950; all regions closed.
2024-11-09T21:22:35,193 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:35,193 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:35,193 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:35,194 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:35,194 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:35,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741834_1010 (size=3066)
2024-11-09T21:22:35,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741834_1010 (size=3066)
2024-11-09T21:22:35,203 DEBUG [RS:0;d86ff69d6719:46221 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/oldWALs
2024-11-09T21:22:35,204 INFO  [RS:0;d86ff69d6719:46221 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d86ff69d6719%2C46221%2C1731187259950.meta:.meta(num 1731187261866)
2024-11-09T21:22:35,204 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:35,204 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:35,205 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:35,205 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:35,205 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:35,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741847_1023 (size=12695)
2024-11-09T21:22:35,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741847_1023 (size=12695)
2024-11-09T21:22:35,210 INFO  [regionserver/d86ff69d6719:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-09T21:22:35,212 DEBUG [RS:0;d86ff69d6719:46221 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/oldWALs
2024-11-09T21:22:35,212 INFO  [RS:0;d86ff69d6719:46221 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d86ff69d6719%2C46221%2C1731187259950:(num 1731187334857)
2024-11-09T21:22:35,212 DEBUG [RS:0;d86ff69d6719:46221 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:22:35,212 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.LeaseManager(133): Closed leases
2024-11-09T21:22:35,212 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-09T21:22:35,212 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.ChoreService(370): Chore service for: regionserver/d86ff69d6719:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-09T21:22:35,213 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-09T21:22:35,213 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-09T21:22:35,213 INFO  [RS:0;d86ff69d6719:46221 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46221
2024-11-09T21:22:35,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T21:22:35,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d86ff69d6719,46221,1731187259950
2024-11-09T21:22:35,244 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-09T21:22:35,245 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d86ff69d6719,46221,1731187259950]
2024-11-09T21:22:35,268 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d86ff69d6719,46221,1731187259950 already deleted, retry=false
2024-11-09T21:22:35,268 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d86ff69d6719,46221,1731187259950 expired; onlineServers=0
2024-11-09T21:22:35,269 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd86ff69d6719,46577,1731187259225' *****
2024-11-09T21:22:35,269 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-09T21:22:35,269 INFO  [M:0;d86ff69d6719:46577 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-09T21:22:35,269 INFO  [M:0;d86ff69d6719:46577 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-09T21:22:35,270 DEBUG [M:0;d86ff69d6719:46577 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-09T21:22:35,270 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-09T21:22:35,270 DEBUG [M:0;d86ff69d6719:46577 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-09T21:22:35,270 DEBUG [master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187261011 {}] cleaner.HFileCleaner(306): Exit Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187261011,5,FailOnTimeoutGroup]
2024-11-09T21:22:35,270 DEBUG [master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187261009 {}] cleaner.HFileCleaner(306): Exit Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187261009,5,FailOnTimeoutGroup]
2024-11-09T21:22:35,270 INFO  [M:0;d86ff69d6719:46577 {}] hbase.ChoreService(370): Chore service for: master/d86ff69d6719:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-09T21:22:35,271 INFO  [M:0;d86ff69d6719:46577 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-09T21:22:35,271 DEBUG [M:0;d86ff69d6719:46577 {}] master.HMaster(1795): Stopping service threads
2024-11-09T21:22:35,271 INFO  [M:0;d86ff69d6719:46577 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-09T21:22:35,271 INFO  [M:0;d86ff69d6719:46577 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-09T21:22:35,273 INFO  [M:0;d86ff69d6719:46577 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-09T21:22:35,273 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-09T21:22:35,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-09T21:22:35,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:35,276 DEBUG [M:0;d86ff69d6719:46577 {}] zookeeper.ZKUtil(347): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-09T21:22:35,276 WARN  [M:0;d86ff69d6719:46577 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-09T21:22:35,277 INFO  [M:0;d86ff69d6719:46577 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/.lastflushedseqids
2024-11-09T21:22:35,291 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741854_1030 (size=130)
2024-11-09T21:22:35,291 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741854_1030 (size=130)
2024-11-09T21:22:35,292 INFO  [M:0;d86ff69d6719:46577 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-09T21:22:35,292 INFO  [M:0;d86ff69d6719:46577 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-09T21:22:35,292 DEBUG [M:0;d86ff69d6719:46577 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-09T21:22:35,292 INFO  [M:0;d86ff69d6719:46577 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:22:35,292 DEBUG [M:0;d86ff69d6719:46577 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:22:35,292 DEBUG [M:0;d86ff69d6719:46577 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-09T21:22:35,292 DEBUG [M:0;d86ff69d6719:46577 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:22:35,292 INFO  [M:0;d86ff69d6719:46577 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.04 KB heapSize=29.21 KB
2024-11-09T21:22:35,308 DEBUG [M:0;d86ff69d6719:46577 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/713f156f81f3472eb6fe443b82229d1a is 82, key is hbase:meta,,1/info:regioninfo/1731187261935/Put/seqid=0
2024-11-09T21:22:35,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741855_1031 (size=5672)
2024-11-09T21:22:35,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741855_1031 (size=5672)
2024-11-09T21:22:35,315 INFO  [M:0;d86ff69d6719:46577 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/713f156f81f3472eb6fe443b82229d1a
2024-11-09T21:22:35,336 DEBUG [M:0;d86ff69d6719:46577 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/61a52b8464a645f69ae1fb53105e5f44 is 767, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731187262764/Put/seqid=0
2024-11-09T21:22:35,342 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741856_1032 (size=6248)
2024-11-09T21:22:35,342 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741856_1032 (size=6248)
2024-11-09T21:22:35,343 INFO  [M:0;d86ff69d6719:46577 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.43 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/61a52b8464a645f69ae1fb53105e5f44
2024-11-09T21:22:35,349 INFO  [M:0;d86ff69d6719:46577 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 61a52b8464a645f69ae1fb53105e5f44
2024-11-09T21:22:35,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:22:35,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x10122c2c6fb0001, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:22:35,360 INFO  [RS:0;d86ff69d6719:46221 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-09T21:22:35,361 INFO  [RS:0;d86ff69d6719:46221 {}] regionserver.HRegionServer(1031): Exiting; stopping=d86ff69d6719,46221,1731187259950; zookeeper connection closed.
2024-11-09T21:22:35,361 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@74a1ab29 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@74a1ab29
2024-11-09T21:22:35,361 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-09T21:22:35,366 DEBUG [M:0;d86ff69d6719:46577 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a56147da8f4547b482aa59387af34c95 is 69, key is d86ff69d6719,46221,1731187259950/rs:state/1731187261115/Put/seqid=0
2024-11-09T21:22:35,371 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741857_1033 (size=5156)
2024-11-09T21:22:35,371 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741857_1033 (size=5156)
2024-11-09T21:22:35,372 INFO  [M:0;d86ff69d6719:46577 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a56147da8f4547b482aa59387af34c95
2024-11-09T21:22:35,392 DEBUG [M:0;d86ff69d6719:46577 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c85cbfebb6af4b66ac5ba8cfde7ac182 is 52, key is load_balancer_on/state:d/1731187262243/Put/seqid=0
2024-11-09T21:22:35,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741858_1034 (size=5056)
2024-11-09T21:22:35,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741858_1034 (size=5056)
2024-11-09T21:22:35,398 INFO  [M:0;d86ff69d6719:46577 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c85cbfebb6af4b66ac5ba8cfde7ac182
2024-11-09T21:22:35,405 DEBUG [M:0;d86ff69d6719:46577 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/713f156f81f3472eb6fe443b82229d1a as hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/713f156f81f3472eb6fe443b82229d1a
2024-11-09T21:22:35,411 INFO  [M:0;d86ff69d6719:46577 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/713f156f81f3472eb6fe443b82229d1a, entries=8, sequenceid=59, filesize=5.5 K
2024-11-09T21:22:35,413 DEBUG [M:0;d86ff69d6719:46577 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/61a52b8464a645f69ae1fb53105e5f44 as hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/61a52b8464a645f69ae1fb53105e5f44
2024-11-09T21:22:35,419 INFO  [M:0;d86ff69d6719:46577 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 61a52b8464a645f69ae1fb53105e5f44
2024-11-09T21:22:35,419 INFO  [M:0;d86ff69d6719:46577 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/61a52b8464a645f69ae1fb53105e5f44, entries=6, sequenceid=59, filesize=6.1 K
2024-11-09T21:22:35,421 DEBUG [M:0;d86ff69d6719:46577 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a56147da8f4547b482aa59387af34c95 as hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a56147da8f4547b482aa59387af34c95
2024-11-09T21:22:35,427 INFO  [M:0;d86ff69d6719:46577 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a56147da8f4547b482aa59387af34c95, entries=1, sequenceid=59, filesize=5.0 K
2024-11-09T21:22:35,428 DEBUG [M:0;d86ff69d6719:46577 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c85cbfebb6af4b66ac5ba8cfde7ac182 as hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c85cbfebb6af4b66ac5ba8cfde7ac182
2024-11-09T21:22:35,435 INFO  [M:0;d86ff69d6719:46577 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c85cbfebb6af4b66ac5ba8cfde7ac182, entries=1, sequenceid=59, filesize=4.9 K
2024-11-09T21:22:35,436 INFO  [M:0;d86ff69d6719:46577 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 144ms, sequenceid=59, compaction requested=false
2024-11-09T21:22:35,438 INFO  [M:0;d86ff69d6719:46577 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:22:35,438 DEBUG [M:0;d86ff69d6719:46577 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731187355292Disabling compacts and flushes for region at 1731187355292Disabling writes for close at 1731187355292Obtaining lock to block concurrent updates at 1731187355292Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731187355292Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23588, getHeapSize=29848, getOffHeapSize=0, getCellsCount=70 at 1731187355293 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731187355293Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731187355294 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731187355308 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731187355308Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731187355321 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731187355335 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731187355335Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731187355349 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731187355365 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731187355365Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731187355378 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731187355391 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731187355391Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7327d6d: reopening flushed file at 1731187355404 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@351065d2: reopening flushed file at 1731187355411 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63bfa791: reopening flushed file at 1731187355419 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9592107: reopening flushed file at 1731187355427 (+8 ms)Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 144ms, sequenceid=59, compaction requested=false at 1731187355436 (+9 ms)Writing region close event to WAL at 1731187355438 (+2 ms)Closed at 1731187355438
2024-11-09T21:22:35,439 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:35,439 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:35,439 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:35,439 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:35,439 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:35,441 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38913 is added to blk_1073741830_1006 (size=27985)
2024-11-09T21:22:35,442 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40763 is added to blk_1073741830_1006 (size=27985)
2024-11-09T21:22:35,442 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-09T21:22:35,442 INFO  [M:0;d86ff69d6719:46577 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-09T21:22:35,443 INFO  [M:0;d86ff69d6719:46577 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46577
2024-11-09T21:22:35,443 INFO  [M:0;d86ff69d6719:46577 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-09T21:22:35,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:22:35,577 INFO  [M:0;d86ff69d6719:46577 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-09T21:22:35,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46577-0x10122c2c6fb0000, quorum=127.0.0.1:50470, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:22:35,586 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:22:35,588 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:22:35,588 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:22:35,588 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:22:35,588 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/hadoop.log.dir/,STOPPED}
2024-11-09T21:22:35,591 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:22:35,591 WARN  [BP-651326839-172.17.0.2-1731187254518 heartbeating to localhost/127.0.0.1:34451 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:22:35,591 WARN  [BP-651326839-172.17.0.2-1731187254518 heartbeating to localhost/127.0.0.1:34451 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-651326839-172.17.0.2-1731187254518 (Datanode Uuid f00a34af-c807-493f-9a4c-e4dc8ac42ce8) service to localhost/127.0.0.1:34451
2024-11-09T21:22:35,591 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:22:35,592 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/cluster_bdeda732-fbb9-6548-402e-8714e08c557f/data/data3/current/BP-651326839-172.17.0.2-1731187254518 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:22:35,592 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/cluster_bdeda732-fbb9-6548-402e-8714e08c557f/data/data4/current/BP-651326839-172.17.0.2-1731187254518 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:22:35,593 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:22:35,595 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:22:35,596 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:22:35,596 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:22:35,596 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:22:35,596 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/hadoop.log.dir/,STOPPED}
2024-11-09T21:22:35,598 WARN  [BP-651326839-172.17.0.2-1731187254518 heartbeating to localhost/127.0.0.1:34451 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:22:35,598 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:22:35,598 WARN  [BP-651326839-172.17.0.2-1731187254518 heartbeating to localhost/127.0.0.1:34451 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-651326839-172.17.0.2-1731187254518 (Datanode Uuid 2e0e4656-3354-4639-8046-462951fa29c3) service to localhost/127.0.0.1:34451
2024-11-09T21:22:35,598 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:22:35,598 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/cluster_bdeda732-fbb9-6548-402e-8714e08c557f/data/data1/current/BP-651326839-172.17.0.2-1731187254518 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:22:35,599 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/cluster_bdeda732-fbb9-6548-402e-8714e08c557f/data/data2/current/BP-651326839-172.17.0.2-1731187254518 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:22:35,599 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:22:35,608 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-09T21:22:35,609 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:22:35,609 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:22:35,609 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:22:35,609 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/hadoop.log.dir/,STOPPED}
2024-11-09T21:22:35,618 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-09T21:22:35,647 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-09T21:22:35,655 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=81 (was 12)
Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34451
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34451 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34451 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@b745d6b
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: master/d86ff69d6719:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-8
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: ForkJoinPool-2-worker-7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: master/d86ff69d6719:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34451
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: regionserver/d86ff69d6719:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:34451
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34451
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:34451
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34451 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=404 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=39 (was 25) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=18628 (was 19086)
2024-11-09T21:22:35,661 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=82, OpenFileDescriptor=404, MaxFileDescriptor=1048576, SystemLoadAverage=39, ProcessCount=11, AvailableMemoryMB=18627
2024-11-09T21:22:35,661 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-09T21:22:35,661 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/hadoop.log.dir so I do NOT create it in target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec
2024-11-09T21:22:35,661 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/546bd869-56f3-3172-e74a-edbf6e70c16e/hadoop.tmp.dir so I do NOT create it in target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec
2024-11-09T21:22:35,661 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/cluster_e398fb7b-503a-57d8-4080-1dddd6f77013, deleteOnExit=true
2024-11-09T21:22:35,661 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-09T21:22:35,662 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/test.cache.data in system properties and HBase conf
2024-11-09T21:22:35,662 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/hadoop.tmp.dir in system properties and HBase conf
2024-11-09T21:22:35,662 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/hadoop.log.dir in system properties and HBase conf
2024-11-09T21:22:35,662 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-09T21:22:35,662 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-09T21:22:35,662 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-09T21:22:35,662 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-09T21:22:35,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-09T21:22:35,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-09T21:22:35,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-09T21:22:35,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-09T21:22:35,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-09T21:22:35,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-09T21:22:35,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-09T21:22:35,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-09T21:22:35,664 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-09T21:22:35,664 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/nfs.dump.dir in system properties and HBase conf
2024-11-09T21:22:35,664 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/java.io.tmpdir in system properties and HBase conf
2024-11-09T21:22:35,664 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-09T21:22:35,664 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-09T21:22:35,664 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-09T21:22:35,677 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-09T21:22:35,964 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:22:35,970 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:22:35,979 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:22:35,979 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:22:35,979 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T21:22:35,980 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:22:35,980 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f841e9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:22:35,980 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30a1c2a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:22:36,076 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7d95bc23{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/java.io.tmpdir/jetty-localhost-45045-hadoop-hdfs-3_4_1-tests_jar-_-any-17837062973159765159/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-09T21:22:36,077 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@78fa6004{HTTP/1.1, (http/1.1)}{localhost:45045}
2024-11-09T21:22:36,077 INFO  [Time-limited test {}] server.Server(415): Started @103496ms
2024-11-09T21:22:36,089 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-09T21:22:36,306 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:22:36,312 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:22:36,313 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:22:36,313 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:22:36,313 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T21:22:36,314 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3e23c0c8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:22:36,314 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7517d9e5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:22:36,406 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7d69c419{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/java.io.tmpdir/jetty-localhost-46697-hadoop-hdfs-3_4_1-tests_jar-_-any-15873596588735315626/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:22:36,407 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2f2378c9{HTTP/1.1, (http/1.1)}{localhost:46697}
2024-11-09T21:22:36,407 INFO  [Time-limited test {}] server.Server(415): Started @103826ms
2024-11-09T21:22:36,408 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:22:36,443 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:22:36,446 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:22:36,447 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:22:36,447 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:22:36,447 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T21:22:36,448 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1ce533a5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:22:36,448 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@aab268d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:22:36,542 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@75434f63{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/java.io.tmpdir/jetty-localhost-32849-hadoop-hdfs-3_4_1-tests_jar-_-any-12175079556997314687/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:22:36,542 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1a2066f8{HTTP/1.1, (http/1.1)}{localhost:32849}
2024-11-09T21:22:36,542 INFO  [Time-limited test {}] server.Server(415): Started @103961ms
2024-11-09T21:22:36,544 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:22:37,094 WARN  [Thread-453 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/cluster_e398fb7b-503a-57d8-4080-1dddd6f77013/data/data1/current/BP-78701540-172.17.0.2-1731187355689/current, will proceed with Du for space computation calculation, 
2024-11-09T21:22:37,094 WARN  [Thread-454 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/cluster_e398fb7b-503a-57d8-4080-1dddd6f77013/data/data2/current/BP-78701540-172.17.0.2-1731187355689/current, will proceed with Du for space computation calculation, 
2024-11-09T21:22:37,114 WARN  [Thread-417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:22:37,117 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xba2eb77170189985 with lease ID 0xb2f7c94e03e5606f: Processing first storage report for DS-0ba1afc0-0a43-441c-ae9a-9abf840e1e33 from datanode DatanodeRegistration(127.0.0.1:35493, datanodeUuid=41c8f95c-2104-450c-8b3b-c6420d1aa605, infoPort=33889, infoSecurePort=0, ipcPort=43621, storageInfo=lv=-57;cid=testClusterID;nsid=1148326095;c=1731187355689)
2024-11-09T21:22:37,117 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xba2eb77170189985 with lease ID 0xb2f7c94e03e5606f: from storage DS-0ba1afc0-0a43-441c-ae9a-9abf840e1e33 node DatanodeRegistration(127.0.0.1:35493, datanodeUuid=41c8f95c-2104-450c-8b3b-c6420d1aa605, infoPort=33889, infoSecurePort=0, ipcPort=43621, storageInfo=lv=-57;cid=testClusterID;nsid=1148326095;c=1731187355689), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:22:37,117 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xba2eb77170189985 with lease ID 0xb2f7c94e03e5606f: Processing first storage report for DS-07e0c4f4-d465-4e67-831e-6c2a123a511a from datanode DatanodeRegistration(127.0.0.1:35493, datanodeUuid=41c8f95c-2104-450c-8b3b-c6420d1aa605, infoPort=33889, infoSecurePort=0, ipcPort=43621, storageInfo=lv=-57;cid=testClusterID;nsid=1148326095;c=1731187355689)
2024-11-09T21:22:37,117 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xba2eb77170189985 with lease ID 0xb2f7c94e03e5606f: from storage DS-07e0c4f4-d465-4e67-831e-6c2a123a511a node DatanodeRegistration(127.0.0.1:35493, datanodeUuid=41c8f95c-2104-450c-8b3b-c6420d1aa605, infoPort=33889, infoSecurePort=0, ipcPort=43621, storageInfo=lv=-57;cid=testClusterID;nsid=1148326095;c=1731187355689), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-09T21:22:37,248 WARN  [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/cluster_e398fb7b-503a-57d8-4080-1dddd6f77013/data/data4/current/BP-78701540-172.17.0.2-1731187355689/current, will proceed with Du for space computation calculation, 
2024-11-09T21:22:37,248 WARN  [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/cluster_e398fb7b-503a-57d8-4080-1dddd6f77013/data/data3/current/BP-78701540-172.17.0.2-1731187355689/current, will proceed with Du for space computation calculation, 
2024-11-09T21:22:37,263 WARN  [Thread-440 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:22:37,266 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9d9cbbf080370bad with lease ID 0xb2f7c94e03e56070: Processing first storage report for DS-5ff344a6-9979-43b8-a1f2-b182a039d8db from datanode DatanodeRegistration(127.0.0.1:35753, datanodeUuid=5d3d7132-68bf-446b-9458-249aa5794aae, infoPort=45197, infoSecurePort=0, ipcPort=33917, storageInfo=lv=-57;cid=testClusterID;nsid=1148326095;c=1731187355689)
2024-11-09T21:22:37,266 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d9cbbf080370bad with lease ID 0xb2f7c94e03e56070: from storage DS-5ff344a6-9979-43b8-a1f2-b182a039d8db node DatanodeRegistration(127.0.0.1:35753, datanodeUuid=5d3d7132-68bf-446b-9458-249aa5794aae, infoPort=45197, infoSecurePort=0, ipcPort=33917, storageInfo=lv=-57;cid=testClusterID;nsid=1148326095;c=1731187355689), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:22:37,266 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9d9cbbf080370bad with lease ID 0xb2f7c94e03e56070: Processing first storage report for DS-bf2816eb-973d-42a6-a590-9efffac204ae from datanode DatanodeRegistration(127.0.0.1:35753, datanodeUuid=5d3d7132-68bf-446b-9458-249aa5794aae, infoPort=45197, infoSecurePort=0, ipcPort=33917, storageInfo=lv=-57;cid=testClusterID;nsid=1148326095;c=1731187355689)
2024-11-09T21:22:37,266 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d9cbbf080370bad with lease ID 0xb2f7c94e03e56070: from storage DS-bf2816eb-973d-42a6-a590-9efffac204ae node DatanodeRegistration(127.0.0.1:35753, datanodeUuid=5d3d7132-68bf-446b-9458-249aa5794aae, infoPort=45197, infoSecurePort=0, ipcPort=33917, storageInfo=lv=-57;cid=testClusterID;nsid=1148326095;c=1731187355689), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:22:37,274 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec
2024-11-09T21:22:37,279 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/cluster_e398fb7b-503a-57d8-4080-1dddd6f77013/zookeeper_0, clientPort=54620, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/cluster_e398fb7b-503a-57d8-4080-1dddd6f77013/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/cluster_e398fb7b-503a-57d8-4080-1dddd6f77013/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-09T21:22:37,280 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54620
2024-11-09T21:22:37,280 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:37,282 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:37,291 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35753 is added to blk_1073741825_1001 (size=7)
2024-11-09T21:22:37,291 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35493 is added to blk_1073741825_1001 (size=7)
2024-11-09T21:22:37,292 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36 with version=8
2024-11-09T21:22:37,293 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/hbase-staging
2024-11-09T21:22:37,295 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d86ff69d6719:0 server-side Connection retries=45
2024-11-09T21:22:37,295 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:22:37,295 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T21:22:37,295 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T21:22:37,295 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:22:37,295 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T21:22:37,295 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-09T21:22:37,295 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T21:22:37,296 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39267
2024-11-09T21:22:37,298 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39267 connecting to ZooKeeper ensemble=127.0.0.1:54620
2024-11-09T21:22:37,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:392670x0, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T21:22:37,368 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39267-0x10122c449210000 connected
2024-11-09T21:22:37,460 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:37,466 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:37,472 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:22:37,472 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36, hbase.cluster.distributed=false
2024-11-09T21:22:37,474 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T21:22:37,474 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39267
2024-11-09T21:22:37,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39267
2024-11-09T21:22:37,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39267
2024-11-09T21:22:37,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39267
2024-11-09T21:22:37,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39267
2024-11-09T21:22:37,493 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d86ff69d6719:0 server-side Connection retries=45
2024-11-09T21:22:37,493 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:22:37,493 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T21:22:37,493 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T21:22:37,493 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:22:37,493 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T21:22:37,493 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-09T21:22:37,493 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T21:22:37,494 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38347
2024-11-09T21:22:37,495 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38347 connecting to ZooKeeper ensemble=127.0.0.1:54620
2024-11-09T21:22:37,496 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:37,498 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:37,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:383470x0, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T21:22:37,509 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38347-0x10122c449210001 connected
2024-11-09T21:22:37,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:22:37,509 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-09T21:22:37,510 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-09T21:22:37,510 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-09T21:22:37,512 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T21:22:37,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38347
2024-11-09T21:22:37,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38347
2024-11-09T21:22:37,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38347
2024-11-09T21:22:37,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38347
2024-11-09T21:22:37,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38347
2024-11-09T21:22:37,525 DEBUG [M:0;d86ff69d6719:39267 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d86ff69d6719:39267
2024-11-09T21:22:37,525 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d86ff69d6719,39267,1731187357294
2024-11-09T21:22:37,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:22:37,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:22:37,534 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d86ff69d6719,39267,1731187357294
2024-11-09T21:22:37,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-09T21:22:37,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:37,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:37,542 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-09T21:22:37,543 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d86ff69d6719,39267,1731187357294 from backup master directory
2024-11-09T21:22:37,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d86ff69d6719,39267,1731187357294
2024-11-09T21:22:37,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:22:37,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:22:37,550 WARN  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T21:22:37,550 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d86ff69d6719,39267,1731187357294
2024-11-09T21:22:37,555 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/hbase.id] with ID: b056d87b-b18f-45b3-8e07-e1fefa1d6db8
2024-11-09T21:22:37,555 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/.tmp/hbase.id
2024-11-09T21:22:37,562 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35493 is added to blk_1073741826_1002 (size=42)
2024-11-09T21:22:37,562 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35753 is added to blk_1073741826_1002 (size=42)
2024-11-09T21:22:37,563 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/.tmp/hbase.id]:[hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/hbase.id]
2024-11-09T21:22:37,582 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:37,582 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-09T21:22:37,584 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-09T21:22:37,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:37,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:37,598 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35493 is added to blk_1073741827_1003 (size=196)
2024-11-09T21:22:37,599 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35753 is added to blk_1073741827_1003 (size=196)
2024-11-09T21:22:37,600 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-09T21:22:37,601 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-09T21:22:37,601 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:22:37,611 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35493 is added to blk_1073741828_1004 (size=1189)
2024-11-09T21:22:37,611 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35753 is added to blk_1073741828_1004 (size=1189)
2024-11-09T21:22:37,612 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store
2024-11-09T21:22:37,622 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35753 is added to blk_1073741829_1005 (size=34)
2024-11-09T21:22:37,623 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35493 is added to blk_1073741829_1005 (size=34)
2024-11-09T21:22:37,623 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:22:37,623 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-09T21:22:37,623 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:22:37,623 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:22:37,624 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-09T21:22:37,624 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:22:37,624 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:22:37,624 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731187357623Disabling compacts and flushes for region at 1731187357623Disabling writes for close at 1731187357624 (+1 ms)Writing region close event to WAL at 1731187357624Closed at 1731187357624
2024-11-09T21:22:37,625 WARN  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/.initializing
2024-11-09T21:22:37,625 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/WALs/d86ff69d6719,39267,1731187357294
2024-11-09T21:22:37,628 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C39267%2C1731187357294, suffix=, logDir=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/WALs/d86ff69d6719,39267,1731187357294, archiveDir=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/oldWALs, maxLogs=10
2024-11-09T21:22:37,628 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C39267%2C1731187357294.1731187357628
2024-11-09T21:22:37,653 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/WALs/d86ff69d6719,39267,1731187357294/d86ff69d6719%2C39267%2C1731187357294.1731187357628
2024-11-09T21:22:37,655 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33889:33889),(127.0.0.1/127.0.0.1:45197:45197)]
2024-11-09T21:22:37,658 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:22:37,658 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:22:37,658 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:22:37,658 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:22:37,660 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:22:37,662 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-09T21:22:37,662 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:37,662 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:37,662 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:22:37,664 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-09T21:22:37,664 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:37,664 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:22:37,665 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:22:37,667 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-09T21:22:37,667 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:37,667 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:22:37,667 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:22:37,669 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-09T21:22:37,669 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:37,669 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:22:37,669 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:22:37,670 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:22:37,671 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:22:37,672 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:22:37,672 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:22:37,673 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-09T21:22:37,674 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:22:37,677 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:22:37,677 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=760083, jitterRate=-0.03350481390953064}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-09T21:22:37,678 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731187357659Initializing all the Stores at 1731187357660 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187357660Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187357660Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187357660Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187357660Cleaning up temporary data from old regions at 1731187357672 (+12 ms)Region opened successfully at 1731187357678 (+6 ms)
2024-11-09T21:22:37,679 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-09T21:22:37,683 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aad27d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0
2024-11-09T21:22:37,684 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-09T21:22:37,684 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-09T21:22:37,684 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-09T21:22:37,684 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-09T21:22:37,685 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-09T21:22:37,685 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-09T21:22:37,685 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-09T21:22:37,687 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-09T21:22:37,688 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-09T21:22:37,700 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-09T21:22:37,700 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-09T21:22:37,701 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-09T21:22:37,708 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-09T21:22:37,708 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-09T21:22:37,709 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-09T21:22:37,716 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-09T21:22:37,717 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-09T21:22:37,725 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-09T21:22:37,727 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-09T21:22:37,733 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-09T21:22:37,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-09T21:22:37,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-09T21:22:37,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:37,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:37,743 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d86ff69d6719,39267,1731187357294, sessionid=0x10122c449210000, setting cluster-up flag (Was=false)
2024-11-09T21:22:37,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:37,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:37,783 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-09T21:22:37,786 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d86ff69d6719,39267,1731187357294
2024-11-09T21:22:37,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:37,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:37,825 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-09T21:22:37,830 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d86ff69d6719,39267,1731187357294
2024-11-09T21:22:37,832 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-09T21:22:37,836 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-09T21:22:37,837 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-09T21:22:37,837 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-09T21:22:37,837 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d86ff69d6719,39267,1731187357294
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-09T21:22:37,841 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:22:37,841 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:22:37,841 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:22:37,841 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:22:37,841 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d86ff69d6719:0, corePoolSize=10, maxPoolSize=10
2024-11-09T21:22:37,841 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:37,841 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2
2024-11-09T21:22:37,842 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:37,842 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731187387842
2024-11-09T21:22:37,843 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-09T21:22:37,843 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-09T21:22:37,843 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-09T21:22:37,843 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-09T21:22:37,843 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-09T21:22:37,843 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-09T21:22:37,843 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:37,844 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-09T21:22:37,844 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-09T21:22:37,844 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:22:37,844 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-09T21:22:37,844 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-09T21:22:37,844 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-09T21:22:37,844 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-09T21:22:37,844 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187357844,5,FailOnTimeoutGroup]
2024-11-09T21:22:37,844 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187357844,5,FailOnTimeoutGroup]
2024-11-09T21:22:37,845 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:37,845 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-09T21:22:37,845 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:37,845 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:37,845 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:37,845 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-09T21:22:37,853 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35493 is added to blk_1073741831_1007 (size=1321)
2024-11-09T21:22:37,853 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35753 is added to blk_1073741831_1007 (size=1321)
2024-11-09T21:22:37,854 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-09T21:22:37,854 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36
2024-11-09T21:22:37,862 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35753 is added to blk_1073741832_1008 (size=32)
2024-11-09T21:22:37,862 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35493 is added to blk_1073741832_1008 (size=32)
2024-11-09T21:22:37,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:22:37,866 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-09T21:22:37,868 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-09T21:22:37,869 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:37,869 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:37,869 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-09T21:22:37,871 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-09T21:22:37,871 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:37,872 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:37,872 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-09T21:22:37,874 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-09T21:22:37,874 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:37,875 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:37,875 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-09T21:22:37,876 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-09T21:22:37,876 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:37,877 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:37,877 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-09T21:22:37,878 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/data/hbase/meta/1588230740
2024-11-09T21:22:37,878 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/data/hbase/meta/1588230740
2024-11-09T21:22:37,880 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-09T21:22:37,880 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-09T21:22:37,880 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-09T21:22:37,881 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-09T21:22:37,884 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:22:37,885 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=735985, jitterRate=-0.06414762139320374}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-09T21:22:37,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731187357863Initializing all the Stores at 1731187357864 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187357864Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187357866 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187357866Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187357866Cleaning up temporary data from old regions at 1731187357880 (+14 ms)Region opened successfully at 1731187357886 (+6 ms)
2024-11-09T21:22:37,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-09T21:22:37,886 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-09T21:22:37,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-09T21:22:37,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-09T21:22:37,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-09T21:22:37,887 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-09T21:22:37,887 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731187357886Disabling compacts and flushes for region at 1731187357886Disabling writes for close at 1731187357886Writing region close event to WAL at 1731187357887 (+1 ms)Closed at 1731187357887
2024-11-09T21:22:37,888 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:22:37,888 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-09T21:22:37,889 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-09T21:22:37,890 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-09T21:22:37,892 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-09T21:22:37,916 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.HRegionServer(746): ClusterId : b056d87b-b18f-45b3-8e07-e1fefa1d6db8
2024-11-09T21:22:37,916 DEBUG [RS:0;d86ff69d6719:38347 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-09T21:22:37,950 DEBUG [RS:0;d86ff69d6719:38347 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-09T21:22:37,951 DEBUG [RS:0;d86ff69d6719:38347 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-09T21:22:37,959 DEBUG [RS:0;d86ff69d6719:38347 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-09T21:22:37,959 DEBUG [RS:0;d86ff69d6719:38347 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@798c13f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0
2024-11-09T21:22:37,971 DEBUG [RS:0;d86ff69d6719:38347 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d86ff69d6719:38347
2024-11-09T21:22:37,971 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-09T21:22:37,971 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-09T21:22:37,971 DEBUG [RS:0;d86ff69d6719:38347 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-09T21:22:37,972 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.HRegionServer(2659): reportForDuty to master=d86ff69d6719,39267,1731187357294 with port=38347, startcode=1731187357492
2024-11-09T21:22:37,972 DEBUG [RS:0;d86ff69d6719:38347 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-09T21:22:37,975 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52045, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-11-09T21:22:37,976 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39267 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d86ff69d6719,38347,1731187357492
2024-11-09T21:22:37,976 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39267 {}] master.ServerManager(517): Registering regionserver=d86ff69d6719,38347,1731187357492
2024-11-09T21:22:37,978 DEBUG [RS:0;d86ff69d6719:38347 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36
2024-11-09T21:22:37,978 DEBUG [RS:0;d86ff69d6719:38347 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35901
2024-11-09T21:22:37,978 DEBUG [RS:0;d86ff69d6719:38347 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-09T21:22:37,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T21:22:37,984 DEBUG [RS:0;d86ff69d6719:38347 {}] zookeeper.ZKUtil(111): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d86ff69d6719,38347,1731187357492
2024-11-09T21:22:37,984 WARN  [RS:0;d86ff69d6719:38347 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T21:22:37,984 INFO  [RS:0;d86ff69d6719:38347 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:22:37,984 DEBUG [RS:0;d86ff69d6719:38347 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/WALs/d86ff69d6719,38347,1731187357492
2024-11-09T21:22:37,984 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d86ff69d6719,38347,1731187357492]
2024-11-09T21:22:37,988 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-09T21:22:37,990 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-09T21:22:37,990 INFO  [RS:0;d86ff69d6719:38347 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-09T21:22:37,990 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:37,990 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-09T21:22:37,991 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-09T21:22:37,991 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:37,991 DEBUG [RS:0;d86ff69d6719:38347 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:37,991 DEBUG [RS:0;d86ff69d6719:38347 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:37,991 DEBUG [RS:0;d86ff69d6719:38347 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:37,991 DEBUG [RS:0;d86ff69d6719:38347 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:37,991 DEBUG [RS:0;d86ff69d6719:38347 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:37,991 DEBUG [RS:0;d86ff69d6719:38347 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2
2024-11-09T21:22:37,992 DEBUG [RS:0;d86ff69d6719:38347 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:37,992 DEBUG [RS:0;d86ff69d6719:38347 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:37,992 DEBUG [RS:0;d86ff69d6719:38347 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:37,992 DEBUG [RS:0;d86ff69d6719:38347 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:37,992 DEBUG [RS:0;d86ff69d6719:38347 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:37,992 DEBUG [RS:0;d86ff69d6719:38347 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:37,992 DEBUG [RS:0;d86ff69d6719:38347 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3
2024-11-09T21:22:37,992 DEBUG [RS:0;d86ff69d6719:38347 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3
2024-11-09T21:22:37,992 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:37,992 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:37,992 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:37,993 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:37,993 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:37,993 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,38347,1731187357492-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T21:22:38,007 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-09T21:22:38,007 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,38347,1731187357492-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:38,007 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:38,007 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.Replication(171): d86ff69d6719,38347,1731187357492 started
2024-11-09T21:22:38,019 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:38,019 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.HRegionServer(1482): Serving as d86ff69d6719,38347,1731187357492, RpcServer on d86ff69d6719/172.17.0.2:38347, sessionid=0x10122c449210001
2024-11-09T21:22:38,020 DEBUG [RS:0;d86ff69d6719:38347 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-09T21:22:38,020 DEBUG [RS:0;d86ff69d6719:38347 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d86ff69d6719,38347,1731187357492
2024-11-09T21:22:38,020 DEBUG [RS:0;d86ff69d6719:38347 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,38347,1731187357492'
2024-11-09T21:22:38,020 DEBUG [RS:0;d86ff69d6719:38347 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-09T21:22:38,020 DEBUG [RS:0;d86ff69d6719:38347 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-09T21:22:38,021 DEBUG [RS:0;d86ff69d6719:38347 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-09T21:22:38,021 DEBUG [RS:0;d86ff69d6719:38347 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-09T21:22:38,021 DEBUG [RS:0;d86ff69d6719:38347 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d86ff69d6719,38347,1731187357492
2024-11-09T21:22:38,021 DEBUG [RS:0;d86ff69d6719:38347 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,38347,1731187357492'
2024-11-09T21:22:38,021 DEBUG [RS:0;d86ff69d6719:38347 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-09T21:22:38,022 DEBUG [RS:0;d86ff69d6719:38347 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-09T21:22:38,022 DEBUG [RS:0;d86ff69d6719:38347 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-09T21:22:38,022 INFO  [RS:0;d86ff69d6719:38347 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-09T21:22:38,022 INFO  [RS:0;d86ff69d6719:38347 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-09T21:22:38,042 WARN  [d86ff69d6719:39267 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-09T21:22:38,126 INFO  [RS:0;d86ff69d6719:38347 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C38347%2C1731187357492, suffix=, logDir=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/WALs/d86ff69d6719,38347,1731187357492, archiveDir=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/oldWALs, maxLogs=32
2024-11-09T21:22:38,130 INFO  [RS:0;d86ff69d6719:38347 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C38347%2C1731187357492.1731187358130
2024-11-09T21:22:38,139 INFO  [RS:0;d86ff69d6719:38347 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/WALs/d86ff69d6719,38347,1731187357492/d86ff69d6719%2C38347%2C1731187357492.1731187358130
2024-11-09T21:22:38,142 DEBUG [RS:0;d86ff69d6719:38347 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45197:45197),(127.0.0.1/127.0.0.1:33889:33889)]
2024-11-09T21:22:38,292 DEBUG [d86ff69d6719:39267 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-09T21:22:38,293 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d86ff69d6719,38347,1731187357492
2024-11-09T21:22:38,295 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d86ff69d6719,38347,1731187357492, state=OPENING
2024-11-09T21:22:38,325 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-09T21:22:38,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:38,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:38,378 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:22:38,378 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-09T21:22:38,378 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:22:38,379 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d86ff69d6719,38347,1731187357492}]
2024-11-09T21:22:38,537 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-09T21:22:38,542 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49959, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-09T21:22:38,549 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-09T21:22:38,549 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:22:38,552 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C38347%2C1731187357492.meta, suffix=.meta, logDir=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/WALs/d86ff69d6719,38347,1731187357492, archiveDir=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/oldWALs, maxLogs=32
2024-11-09T21:22:38,555 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C38347%2C1731187357492.meta.1731187358554.meta
2024-11-09T21:22:38,561 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/WALs/d86ff69d6719,38347,1731187357492/d86ff69d6719%2C38347%2C1731187357492.meta.1731187358554.meta
2024-11-09T21:22:38,564 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33889:33889),(127.0.0.1/127.0.0.1:45197:45197)]
2024-11-09T21:22:38,566 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:22:38,566 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-09T21:22:38,566 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-09T21:22:38,566 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-09T21:22:38,567 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-09T21:22:38,567 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:22:38,567 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-09T21:22:38,567 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-09T21:22:38,569 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-09T21:22:38,570 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-09T21:22:38,570 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:38,571 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:38,571 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-09T21:22:38,572 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-09T21:22:38,572 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:38,573 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:38,573 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-09T21:22:38,574 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-09T21:22:38,574 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:38,575 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:38,575 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-09T21:22:38,576 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-09T21:22:38,576 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:38,577 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:38,577 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-09T21:22:38,579 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/data/hbase/meta/1588230740
2024-11-09T21:22:38,581 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/data/hbase/meta/1588230740
2024-11-09T21:22:38,583 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-09T21:22:38,583 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-09T21:22:38,583 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-09T21:22:38,585 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-09T21:22:38,586 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=822832, jitterRate=0.046286165714263916}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-09T21:22:38,586 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-09T21:22:38,587 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731187358567Writing region info on filesystem at 1731187358567Initializing all the Stores at 1731187358569 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187358569Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187358569Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187358569Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187358569Cleaning up temporary data from old regions at 1731187358583 (+14 ms)Running coprocessor post-open hooks at 1731187358586 (+3 ms)Region opened successfully at 1731187358587 (+1 ms)
2024-11-09T21:22:38,588 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731187358536
2024-11-09T21:22:38,591 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-09T21:22:38,591 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-09T21:22:38,592 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,38347,1731187357492
2024-11-09T21:22:38,593 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d86ff69d6719,38347,1731187357492, state=OPEN
2024-11-09T21:22:38,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-09T21:22:38,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-09T21:22:38,639 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d86ff69d6719,38347,1731187357492
2024-11-09T21:22:38,639 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:22:38,639 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:22:38,642 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-09T21:22:38,642 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d86ff69d6719,38347,1731187357492 in 261 msec
2024-11-09T21:22:38,645 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-09T21:22:38,645 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 753 msec
2024-11-09T21:22:38,646 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:22:38,646 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-09T21:22:38,648 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-09T21:22:38,648 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,38347,1731187357492, seqNum=-1]
2024-11-09T21:22:38,648 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T21:22:38,650 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59111, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T21:22:38,657 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 821 msec
2024-11-09T21:22:38,657 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731187358657, completionTime=-1
2024-11-09T21:22:38,657 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-09T21:22:38,657 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-09T21:22:38,659 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-09T21:22:38,659 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731187418659
2024-11-09T21:22:38,660 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731187478660
2024-11-09T21:22:38,660 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-09T21:22:38,660 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,39267,1731187357294-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:38,660 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,39267,1731187357294-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:38,660 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,39267,1731187357294-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:38,660 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d86ff69d6719:39267, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:38,660 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:38,661 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:38,663 DEBUG [master/d86ff69d6719:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-09T21:22:38,666 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.115sec
2024-11-09T21:22:38,666 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-09T21:22:38,666 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-09T21:22:38,666 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-09T21:22:38,666 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-09T21:22:38,666 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-09T21:22:38,666 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,39267,1731187357294-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T21:22:38,666 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,39267,1731187357294-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-09T21:22:38,669 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-09T21:22:38,669 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-09T21:22:38,669 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,39267,1731187357294-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:38,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aa3a1e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:22:38,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d86ff69d6719,39267,-1 for getting cluster id
2024-11-09T21:22:38,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-09T21:22:38,719 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b056d87b-b18f-45b3-8e07-e1fefa1d6db8'
2024-11-09T21:22:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-09T21:22:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b056d87b-b18f-45b3-8e07-e1fefa1d6db8"

2024-11-09T21:22:38,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5756f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:22:38,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d86ff69d6719,39267,-1]
2024-11-09T21:22:38,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-09T21:22:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:22:38,724 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-09T21:22:38,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ca969, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:22:38,725 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-09T21:22:38,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,38347,1731187357492, seqNum=-1]
2024-11-09T21:22:38,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T21:22:38,729 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T21:22:38,731 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d86ff69d6719,39267,1731187357294
2024-11-09T21:22:38,731 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:38,734 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-09T21:22:38,734 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-09T21:22:38,734 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-09T21:22:38,734 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:22:38,734 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:22:38,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:22:38,735 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-09T21:22:38,735 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-09T21:22:38,735 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1434811321, stopped=false
2024-11-09T21:22:38,735 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d86ff69d6719,39267,1731187357294
2024-11-09T21:22:38,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-09T21:22:38,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-09T21:22:38,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:38,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:38,750 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-09T21:22:38,750 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-09T21:22:38,750 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:22:38,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:22:38,750 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:22:38,750 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:22:38,751 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd86ff69d6719,38347,1731187357492' *****
2024-11-09T21:22:38,751 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-09T21:22:38,751 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-09T21:22:38,751 INFO  [RS:0;d86ff69d6719:38347 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-09T21:22:38,751 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-09T21:22:38,751 INFO  [RS:0;d86ff69d6719:38347 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-09T21:22:38,751 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.HRegionServer(959): stopping server d86ff69d6719,38347,1731187357492
2024-11-09T21:22:38,751 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-09T21:22:38,751 INFO  [RS:0;d86ff69d6719:38347 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d86ff69d6719:38347.
2024-11-09T21:22:38,751 DEBUG [RS:0;d86ff69d6719:38347 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:22:38,751 DEBUG [RS:0;d86ff69d6719:38347 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:22:38,752 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-09T21:22:38,752 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-09T21:22:38,752 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-09T21:22:38,752 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-09T21:22:38,752 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-09T21:22:38,752 DEBUG [RS:0;d86ff69d6719:38347 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-09T21:22:38,752 DEBUG [RS:0;d86ff69d6719:38347 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-09T21:22:38,752 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-09T21:22:38,752 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-09T21:22:38,752 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-09T21:22:38,752 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-09T21:22:38,752 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-09T21:22:38,753 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-09T21:22:38,768 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/data/hbase/meta/1588230740/.tmp/ns/086b4c514ab34908bf3f06b333526551 is 43, key is default/ns:d/1731187358650/Put/seqid=0
2024-11-09T21:22:38,774 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35493 is added to blk_1073741835_1011 (size=5153)
2024-11-09T21:22:38,774 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35753 is added to blk_1073741835_1011 (size=5153)
2024-11-09T21:22:38,774 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/data/hbase/meta/1588230740/.tmp/ns/086b4c514ab34908bf3f06b333526551
2024-11-09T21:22:38,782 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/data/hbase/meta/1588230740/.tmp/ns/086b4c514ab34908bf3f06b333526551 as hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/data/hbase/meta/1588230740/ns/086b4c514ab34908bf3f06b333526551
2024-11-09T21:22:38,789 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/data/hbase/meta/1588230740/ns/086b4c514ab34908bf3f06b333526551, entries=2, sequenceid=6, filesize=5.0 K
2024-11-09T21:22:38,790 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 38ms, sequenceid=6, compaction requested=false
2024-11-09T21:22:38,790 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-09T21:22:38,796 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-09T21:22:38,797 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-09T21:22:38,797 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-09T21:22:38,797 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731187358752Running coprocessor pre-close hooks at 1731187358752Disabling compacts and flushes for region at 1731187358752Disabling writes for close at 1731187358752Obtaining lock to block concurrent updates at 1731187358753 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731187358753Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731187358753Flushing stores of hbase:meta,,1.1588230740 at 1731187358754 (+1 ms)Flushing 1588230740/ns: creating writer at 1731187358754Flushing 1588230740/ns: appending metadata at 1731187358768 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731187358768Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70c2fae1: reopening flushed file at 1731187358781 (+13 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 38ms, sequenceid=6, compaction requested=false at 1731187358790 (+9 ms)Writing region close event to WAL at 1731187358792 (+2 ms)Running coprocessor post-close hooks at 1731187358797 (+5 ms)Closed at 1731187358797
2024-11-09T21:22:38,797 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-09T21:22:38,952 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.HRegionServer(976): stopping server d86ff69d6719,38347,1731187357492; all regions closed.
2024-11-09T21:22:38,953 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:38,953 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:38,954 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:38,954 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:38,954 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:38,958 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35753 is added to blk_1073741834_1010 (size=1152)
2024-11-09T21:22:38,958 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35493 is added to blk_1073741834_1010 (size=1152)
2024-11-09T21:22:38,963 DEBUG [RS:0;d86ff69d6719:38347 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/oldWALs
2024-11-09T21:22:38,963 INFO  [RS:0;d86ff69d6719:38347 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d86ff69d6719%2C38347%2C1731187357492.meta:.meta(num 1731187358554)
2024-11-09T21:22:38,964 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:38,964 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:38,964 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:38,964 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:38,965 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:38,967 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35493 is added to blk_1073741833_1009 (size=93)
2024-11-09T21:22:38,967 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35753 is added to blk_1073741833_1009 (size=93)
2024-11-09T21:22:38,971 DEBUG [RS:0;d86ff69d6719:38347 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/oldWALs
2024-11-09T21:22:38,971 INFO  [RS:0;d86ff69d6719:38347 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d86ff69d6719%2C38347%2C1731187357492:(num 1731187358130)
2024-11-09T21:22:38,971 DEBUG [RS:0;d86ff69d6719:38347 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:22:38,971 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.LeaseManager(133): Closed leases
2024-11-09T21:22:38,971 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-09T21:22:38,971 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.ChoreService(370): Chore service for: regionserver/d86ff69d6719:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-09T21:22:38,972 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-09T21:22:38,972 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-09T21:22:38,972 INFO  [RS:0;d86ff69d6719:38347 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38347
2024-11-09T21:22:39,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T21:22:39,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d86ff69d6719,38347,1731187357492
2024-11-09T21:22:39,000 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-09T21:22:39,000 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d86ff69d6719,38347,1731187357492]
2024-11-09T21:22:39,016 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d86ff69d6719,38347,1731187357492 already deleted, retry=false
2024-11-09T21:22:39,016 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d86ff69d6719,38347,1731187357492 expired; onlineServers=0
2024-11-09T21:22:39,016 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd86ff69d6719,39267,1731187357294' *****
2024-11-09T21:22:39,016 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-09T21:22:39,017 INFO  [M:0;d86ff69d6719:39267 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-09T21:22:39,017 INFO  [M:0;d86ff69d6719:39267 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-09T21:22:39,017 DEBUG [M:0;d86ff69d6719:39267 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-09T21:22:39,017 DEBUG [master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187357844 {}] cleaner.HFileCleaner(306): Exit Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187357844,5,FailOnTimeoutGroup]
2024-11-09T21:22:39,018 DEBUG [M:0;d86ff69d6719:39267 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-09T21:22:39,018 DEBUG [master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187357844 {}] cleaner.HFileCleaner(306): Exit Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187357844,5,FailOnTimeoutGroup]
2024-11-09T21:22:39,018 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-09T21:22:39,018 INFO  [M:0;d86ff69d6719:39267 {}] hbase.ChoreService(370): Chore service for: master/d86ff69d6719:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-09T21:22:39,018 INFO  [M:0;d86ff69d6719:39267 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-09T21:22:39,018 DEBUG [M:0;d86ff69d6719:39267 {}] master.HMaster(1795): Stopping service threads
2024-11-09T21:22:39,019 INFO  [M:0;d86ff69d6719:39267 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-09T21:22:39,019 INFO  [M:0;d86ff69d6719:39267 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-09T21:22:39,019 INFO  [M:0;d86ff69d6719:39267 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-09T21:22:39,019 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-09T21:22:39,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-09T21:22:39,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:39,025 DEBUG [M:0;d86ff69d6719:39267 {}] zookeeper.ZKUtil(347): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-09T21:22:39,025 WARN  [M:0;d86ff69d6719:39267 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-09T21:22:39,026 INFO  [M:0;d86ff69d6719:39267 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/.lastflushedseqids
2024-11-09T21:22:39,034 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35753 is added to blk_1073741836_1012 (size=99)
2024-11-09T21:22:39,035 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35493 is added to blk_1073741836_1012 (size=99)
2024-11-09T21:22:39,035 INFO  [M:0;d86ff69d6719:39267 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-09T21:22:39,035 INFO  [M:0;d86ff69d6719:39267 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-09T21:22:39,035 DEBUG [M:0;d86ff69d6719:39267 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-09T21:22:39,035 INFO  [M:0;d86ff69d6719:39267 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:22:39,035 DEBUG [M:0;d86ff69d6719:39267 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:22:39,035 DEBUG [M:0;d86ff69d6719:39267 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-09T21:22:39,036 DEBUG [M:0;d86ff69d6719:39267 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:22:39,036 INFO  [M:0;d86ff69d6719:39267 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-09T21:22:39,052 DEBUG [M:0;d86ff69d6719:39267 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0b03fb47fea74d13a53dfe33a94ed18f is 82, key is hbase:meta,,1/info:regioninfo/1731187358592/Put/seqid=0
2024-11-09T21:22:39,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35493 is added to blk_1073741837_1013 (size=5672)
2024-11-09T21:22:39,058 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35753 is added to blk_1073741837_1013 (size=5672)
2024-11-09T21:22:39,058 INFO  [M:0;d86ff69d6719:39267 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0b03fb47fea74d13a53dfe33a94ed18f
2024-11-09T21:22:39,078 DEBUG [M:0;d86ff69d6719:39267 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/38efbe340bf54b3689338f93f8e68c94 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731187358656/Put/seqid=0
2024-11-09T21:22:39,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35493 is added to blk_1073741838_1014 (size=5275)
2024-11-09T21:22:39,083 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35753 is added to blk_1073741838_1014 (size=5275)
2024-11-09T21:22:39,083 INFO  [M:0;d86ff69d6719:39267 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/38efbe340bf54b3689338f93f8e68c94
2024-11-09T21:22:39,101 DEBUG [M:0;d86ff69d6719:39267 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/48fe089cb4124d009852d171c5ebf34b is 69, key is d86ff69d6719,38347,1731187357492/rs:state/1731187357976/Put/seqid=0
2024-11-09T21:22:39,107 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35493 is added to blk_1073741839_1015 (size=5156)
2024-11-09T21:22:39,107 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35753 is added to blk_1073741839_1015 (size=5156)
2024-11-09T21:22:39,107 INFO  [M:0;d86ff69d6719:39267 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/48fe089cb4124d009852d171c5ebf34b
2024-11-09T21:22:39,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:22:39,108 INFO  [RS:0;d86ff69d6719:38347 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-09T21:22:39,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38347-0x10122c449210001, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:22:39,108 INFO  [RS:0;d86ff69d6719:38347 {}] regionserver.HRegionServer(1031): Exiting; stopping=d86ff69d6719,38347,1731187357492; zookeeper connection closed.
2024-11-09T21:22:39,108 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@233b91af {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@233b91af
2024-11-09T21:22:39,108 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-09T21:22:39,127 DEBUG [M:0;d86ff69d6719:39267 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c78c25df22754209a20e2b9e94baa806 is 52, key is load_balancer_on/state:d/1731187358733/Put/seqid=0
2024-11-09T21:22:39,132 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35753 is added to blk_1073741840_1016 (size=5056)
2024-11-09T21:22:39,133 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35493 is added to blk_1073741840_1016 (size=5056)
2024-11-09T21:22:39,133 INFO  [M:0;d86ff69d6719:39267 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c78c25df22754209a20e2b9e94baa806
2024-11-09T21:22:39,140 DEBUG [M:0;d86ff69d6719:39267 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0b03fb47fea74d13a53dfe33a94ed18f as hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0b03fb47fea74d13a53dfe33a94ed18f
2024-11-09T21:22:39,147 INFO  [M:0;d86ff69d6719:39267 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0b03fb47fea74d13a53dfe33a94ed18f, entries=8, sequenceid=29, filesize=5.5 K
2024-11-09T21:22:39,148 DEBUG [M:0;d86ff69d6719:39267 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/38efbe340bf54b3689338f93f8e68c94 as hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/38efbe340bf54b3689338f93f8e68c94
2024-11-09T21:22:39,154 INFO  [M:0;d86ff69d6719:39267 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/38efbe340bf54b3689338f93f8e68c94, entries=3, sequenceid=29, filesize=5.2 K
2024-11-09T21:22:39,155 DEBUG [M:0;d86ff69d6719:39267 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/48fe089cb4124d009852d171c5ebf34b as hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/48fe089cb4124d009852d171c5ebf34b
2024-11-09T21:22:39,161 INFO  [M:0;d86ff69d6719:39267 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/48fe089cb4124d009852d171c5ebf34b, entries=1, sequenceid=29, filesize=5.0 K
2024-11-09T21:22:39,163 DEBUG [M:0;d86ff69d6719:39267 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c78c25df22754209a20e2b9e94baa806 as hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c78c25df22754209a20e2b9e94baa806
2024-11-09T21:22:39,169 INFO  [M:0;d86ff69d6719:39267 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35901/user/jenkins/test-data/4af69751-985b-6dd4-d5d8-ead4ad79ff36/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c78c25df22754209a20e2b9e94baa806, entries=1, sequenceid=29, filesize=4.9 K
2024-11-09T21:22:39,171 INFO  [M:0;d86ff69d6719:39267 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 135ms, sequenceid=29, compaction requested=false
2024-11-09T21:22:39,172 INFO  [M:0;d86ff69d6719:39267 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:22:39,173 DEBUG [M:0;d86ff69d6719:39267 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731187359035Disabling compacts and flushes for region at 1731187359035Disabling writes for close at 1731187359035Obtaining lock to block concurrent updates at 1731187359036 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731187359036Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731187359036Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731187359037 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731187359037Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731187359052 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731187359052Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731187359063 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731187359077 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731187359077Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731187359088 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731187359101 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731187359101Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731187359113 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731187359127 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731187359127Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8410276: reopening flushed file at 1731187359139 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28273265: reopening flushed file at 1731187359147 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16084119: reopening flushed file at 1731187359154 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@153b7242: reopening flushed file at 1731187359162 (+8 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 135ms, sequenceid=29, compaction requested=false at 1731187359171 (+9 ms)Writing region close event to WAL at 1731187359172 (+1 ms)Closed at 1731187359172
2024-11-09T21:22:39,174 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:39,174 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:39,174 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:39,174 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:39,174 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:39,177 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35753 is added to blk_1073741830_1006 (size=10311)
2024-11-09T21:22:39,177 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35493 is added to blk_1073741830_1006 (size=10311)
2024-11-09T21:22:39,177 INFO  [M:0;d86ff69d6719:39267 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-09T21:22:39,177 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-09T21:22:39,178 INFO  [M:0;d86ff69d6719:39267 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39267
2024-11-09T21:22:39,178 INFO  [M:0;d86ff69d6719:39267 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-09T21:22:39,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:22:39,308 INFO  [M:0;d86ff69d6719:39267 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-09T21:22:39,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39267-0x10122c449210000, quorum=127.0.0.1:54620, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:22:39,312 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@75434f63{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:22:39,312 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1a2066f8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:22:39,312 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:22:39,313 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@aab268d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:22:39,313 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1ce533a5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/hadoop.log.dir/,STOPPED}
2024-11-09T21:22:39,314 WARN  [BP-78701540-172.17.0.2-1731187355689 heartbeating to localhost/127.0.0.1:35901 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:22:39,314 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:22:39,314 WARN  [BP-78701540-172.17.0.2-1731187355689 heartbeating to localhost/127.0.0.1:35901 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-78701540-172.17.0.2-1731187355689 (Datanode Uuid 5d3d7132-68bf-446b-9458-249aa5794aae) service to localhost/127.0.0.1:35901
2024-11-09T21:22:39,314 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:22:39,315 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/cluster_e398fb7b-503a-57d8-4080-1dddd6f77013/data/data3/current/BP-78701540-172.17.0.2-1731187355689 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:22:39,315 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/cluster_e398fb7b-503a-57d8-4080-1dddd6f77013/data/data4/current/BP-78701540-172.17.0.2-1731187355689 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:22:39,315 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:22:39,317 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7d69c419{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:22:39,317 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2f2378c9{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:22:39,317 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:22:39,317 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7517d9e5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:22:39,318 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3e23c0c8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/hadoop.log.dir/,STOPPED}
2024-11-09T21:22:39,318 WARN  [BP-78701540-172.17.0.2-1731187355689 heartbeating to localhost/127.0.0.1:35901 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:22:39,319 WARN  [BP-78701540-172.17.0.2-1731187355689 heartbeating to localhost/127.0.0.1:35901 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-78701540-172.17.0.2-1731187355689 (Datanode Uuid 41c8f95c-2104-450c-8b3b-c6420d1aa605) service to localhost/127.0.0.1:35901
2024-11-09T21:22:39,319 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/cluster_e398fb7b-503a-57d8-4080-1dddd6f77013/data/data1/current/BP-78701540-172.17.0.2-1731187355689 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:22:39,319 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/cluster_e398fb7b-503a-57d8-4080-1dddd6f77013/data/data2/current/BP-78701540-172.17.0.2-1731187355689 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:22:39,319 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:22:39,319 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:22:39,319 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:22:39,324 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7d95bc23{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-09T21:22:39,325 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@78fa6004{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:22:39,325 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:22:39,325 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30a1c2a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:22:39,325 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f841e9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/hadoop.log.dir/,STOPPED}
2024-11-09T21:22:39,331 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-09T21:22:39,346 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-09T21:22:39,346 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-09T21:22:39,346 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/hadoop.log.dir so I do NOT create it in target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0
2024-11-09T21:22:39,346 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/611f9314-fad4-d7f6-98d1-d9dfa6c1e8ec/hadoop.tmp.dir so I do NOT create it in target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0
2024-11-09T21:22:39,347 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5, deleteOnExit=true
2024-11-09T21:22:39,347 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-09T21:22:39,347 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/test.cache.data in system properties and HBase conf
2024-11-09T21:22:39,347 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.tmp.dir in system properties and HBase conf
2024-11-09T21:22:39,347 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.log.dir in system properties and HBase conf
2024-11-09T21:22:39,347 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-09T21:22:39,347 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-09T21:22:39,347 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-09T21:22:39,347 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-09T21:22:39,347 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-09T21:22:39,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-09T21:22:39,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-09T21:22:39,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-09T21:22:39,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-09T21:22:39,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-09T21:22:39,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-09T21:22:39,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-09T21:22:39,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-09T21:22:39,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/nfs.dump.dir in system properties and HBase conf
2024-11-09T21:22:39,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/java.io.tmpdir in system properties and HBase conf
2024-11-09T21:22:39,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-09T21:22:39,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-09T21:22:39,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-09T21:22:39,360 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-09T21:22:39,564 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-09T21:22:39,564 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-09T21:22:39,565 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-09T21:22:39,566 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-09T21:22:39,584 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:22:39,589 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:22:39,594 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:22:39,594 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:22:39,594 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-09T21:22:39,595 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:22:39,595 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4ac253d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:22:39,596 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3e26ba04{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:22:39,690 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@75096fee{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/java.io.tmpdir/jetty-localhost-43425-hadoop-hdfs-3_4_1-tests_jar-_-any-16890290074527350275/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-09T21:22:39,691 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@346c0162{HTTP/1.1, (http/1.1)}{localhost:43425}
2024-11-09T21:22:39,691 INFO  [Time-limited test {}] server.Server(415): Started @107109ms
2024-11-09T21:22:39,702 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-09T21:22:39,993 INFO  [regionserver/d86ff69d6719:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-09T21:22:40,061 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:22:40,068 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:22:40,110 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-09T21:22:40,111 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:22:40,127 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:22:40,129 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:22:40,130 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:22:40,144 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:22:40,148 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:22:40,149 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:22:40,149 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:22:40,149 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T21:22:40,150 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d4a4bdf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:22:40,150 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@471af73e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:22:40,244 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7d85f4da{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/java.io.tmpdir/jetty-localhost-41047-hadoop-hdfs-3_4_1-tests_jar-_-any-2661738444969751018/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:22:40,245 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@32d6576d{HTTP/1.1, (http/1.1)}{localhost:41047}
2024-11-09T21:22:40,245 INFO  [Time-limited test {}] server.Server(415): Started @107663ms
2024-11-09T21:22:40,246 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:22:40,277 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:22:40,281 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:22:40,283 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:22:40,283 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:22:40,283 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T21:22:40,284 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@73b17c7e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:22:40,284 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7891c14c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:22:40,376 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1411a57a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/java.io.tmpdir/jetty-localhost-37585-hadoop-hdfs-3_4_1-tests_jar-_-any-13027894324955361937/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:22:40,376 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4495a8b{HTTP/1.1, (http/1.1)}{localhost:37585}
2024-11-09T21:22:40,376 INFO  [Time-limited test {}] server.Server(415): Started @107795ms
2024-11-09T21:22:40,378 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:22:41,003 WARN  [Thread-673 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data1/current/BP-1331531764-172.17.0.2-1731187359371/current, will proceed with Du for space computation calculation, 
2024-11-09T21:22:41,003 WARN  [Thread-674 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data2/current/BP-1331531764-172.17.0.2-1731187359371/current, will proceed with Du for space computation calculation, 
2024-11-09T21:22:41,021 WARN  [Thread-637 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:22:41,024 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb188f158d145f91d with lease ID 0xfd543dc177bc545a: Processing first storage report for DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6 from datanode DatanodeRegistration(127.0.0.1:32917, datanodeUuid=ac5cfbe0-8e00-4177-b0da-f34af1b9eff1, infoPort=34199, infoSecurePort=0, ipcPort=46751, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371)
2024-11-09T21:22:41,024 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb188f158d145f91d with lease ID 0xfd543dc177bc545a: from storage DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6 node DatanodeRegistration(127.0.0.1:32917, datanodeUuid=ac5cfbe0-8e00-4177-b0da-f34af1b9eff1, infoPort=34199, infoSecurePort=0, ipcPort=46751, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:22:41,024 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb188f158d145f91d with lease ID 0xfd543dc177bc545a: Processing first storage report for DS-c6447aba-ab52-4dac-bb47-d65ff8ff1eed from datanode DatanodeRegistration(127.0.0.1:32917, datanodeUuid=ac5cfbe0-8e00-4177-b0da-f34af1b9eff1, infoPort=34199, infoSecurePort=0, ipcPort=46751, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371)
2024-11-09T21:22:41,024 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb188f158d145f91d with lease ID 0xfd543dc177bc545a: from storage DS-c6447aba-ab52-4dac-bb47-d65ff8ff1eed node DatanodeRegistration(127.0.0.1:32917, datanodeUuid=ac5cfbe0-8e00-4177-b0da-f34af1b9eff1, infoPort=34199, infoSecurePort=0, ipcPort=46751, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:22:41,144 WARN  [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data3/current/BP-1331531764-172.17.0.2-1731187359371/current, will proceed with Du for space computation calculation, 
2024-11-09T21:22:41,145 WARN  [Thread-685 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data4/current/BP-1331531764-172.17.0.2-1731187359371/current, will proceed with Du for space computation calculation, 
2024-11-09T21:22:41,166 WARN  [Thread-660 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:22:41,168 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x88a620144ddcae28 with lease ID 0xfd543dc177bc545b: Processing first storage report for DS-246beb66-219b-4473-b90c-a937cc6aa11e from datanode DatanodeRegistration(127.0.0.1:44811, datanodeUuid=2894e6a3-288e-408f-8bd5-b84a909e5593, infoPort=42927, infoSecurePort=0, ipcPort=45395, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371)
2024-11-09T21:22:41,169 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x88a620144ddcae28 with lease ID 0xfd543dc177bc545b: from storage DS-246beb66-219b-4473-b90c-a937cc6aa11e node DatanodeRegistration(127.0.0.1:44811, datanodeUuid=2894e6a3-288e-408f-8bd5-b84a909e5593, infoPort=42927, infoSecurePort=0, ipcPort=45395, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:22:41,169 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x88a620144ddcae28 with lease ID 0xfd543dc177bc545b: Processing first storage report for DS-5a4e1575-1fa4-460a-ac37-4380e7ac604e from datanode DatanodeRegistration(127.0.0.1:44811, datanodeUuid=2894e6a3-288e-408f-8bd5-b84a909e5593, infoPort=42927, infoSecurePort=0, ipcPort=45395, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371)
2024-11-09T21:22:41,169 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x88a620144ddcae28 with lease ID 0xfd543dc177bc545b: from storage DS-5a4e1575-1fa4-460a-ac37-4380e7ac604e node DatanodeRegistration(127.0.0.1:44811, datanodeUuid=2894e6a3-288e-408f-8bd5-b84a909e5593, infoPort=42927, infoSecurePort=0, ipcPort=45395, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:22:41,211 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0
2024-11-09T21:22:41,215 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/zookeeper_0, clientPort=54077, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-09T21:22:41,216 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54077
2024-11-09T21:22:41,216 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:41,219 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:41,228 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44811 is added to blk_1073741825_1001 (size=7)
2024-11-09T21:22:41,228 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32917 is added to blk_1073741825_1001 (size=7)
2024-11-09T21:22:41,229 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a with version=8
2024-11-09T21:22:41,229 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/hbase-staging
2024-11-09T21:22:41,231 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d86ff69d6719:0 server-side Connection retries=45
2024-11-09T21:22:41,231 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:22:41,231 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T21:22:41,231 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T21:22:41,231 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:22:41,231 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T21:22:41,231 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-09T21:22:41,231 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T21:22:41,232 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33537
2024-11-09T21:22:41,234 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33537 connecting to ZooKeeper ensemble=127.0.0.1:54077
2024-11-09T21:22:41,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:335370x0, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T21:22:41,285 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33537-0x10122c458830000 connected
2024-11-09T21:22:41,349 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:41,351 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:41,353 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:22:41,354 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a, hbase.cluster.distributed=false
2024-11-09T21:22:41,356 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T21:22:41,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33537
2024-11-09T21:22:41,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33537
2024-11-09T21:22:41,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33537
2024-11-09T21:22:41,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33537
2024-11-09T21:22:41,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33537
2024-11-09T21:22:41,371 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d86ff69d6719:0 server-side Connection retries=45
2024-11-09T21:22:41,371 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:22:41,371 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T21:22:41,371 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T21:22:41,371 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:22:41,372 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T21:22:41,372 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-09T21:22:41,372 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T21:22:41,372 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37671
2024-11-09T21:22:41,374 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37671 connecting to ZooKeeper ensemble=127.0.0.1:54077
2024-11-09T21:22:41,374 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:41,376 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:41,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:376710x0, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T21:22:41,383 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:376710x0, quorum=127.0.0.1:54077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:22:41,383 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37671-0x10122c458830001 connected
2024-11-09T21:22:41,383 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-09T21:22:41,384 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-09T21:22:41,384 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-09T21:22:41,385 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T21:22:41,386 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37671
2024-11-09T21:22:41,386 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37671
2024-11-09T21:22:41,386 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37671
2024-11-09T21:22:41,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37671
2024-11-09T21:22:41,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37671
2024-11-09T21:22:41,399 DEBUG [M:0;d86ff69d6719:33537 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d86ff69d6719:33537
2024-11-09T21:22:41,399 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d86ff69d6719,33537,1731187361230
2024-11-09T21:22:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:22:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:22:41,408 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d86ff69d6719,33537,1731187361230
2024-11-09T21:22:41,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:41,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-09T21:22:41,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:41,416 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-09T21:22:41,417 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d86ff69d6719,33537,1731187361230 from backup master directory
2024-11-09T21:22:41,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:22:41,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d86ff69d6719,33537,1731187361230
2024-11-09T21:22:41,424 WARN  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T21:22:41,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:22:41,424 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d86ff69d6719,33537,1731187361230
2024-11-09T21:22:41,428 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/hbase.id] with ID: 34ba9df3-8258-4c52-a449-7d875113d2f3
2024-11-09T21:22:41,429 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/.tmp/hbase.id
2024-11-09T21:22:41,435 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32917 is added to blk_1073741826_1002 (size=42)
2024-11-09T21:22:41,435 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44811 is added to blk_1073741826_1002 (size=42)
2024-11-09T21:22:41,436 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/.tmp/hbase.id]:[hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/hbase.id]
2024-11-09T21:22:41,449 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:41,450 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-09T21:22:41,451 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-09T21:22:41,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:41,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:41,465 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32917 is added to blk_1073741827_1003 (size=196)
2024-11-09T21:22:41,465 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44811 is added to blk_1073741827_1003 (size=196)
2024-11-09T21:22:41,466 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-09T21:22:41,467 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-09T21:22:41,467 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:22:41,475 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32917 is added to blk_1073741828_1004 (size=1189)
2024-11-09T21:22:41,475 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44811 is added to blk_1073741828_1004 (size=1189)
2024-11-09T21:22:41,476 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store
2024-11-09T21:22:41,483 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44811 is added to blk_1073741829_1005 (size=34)
2024-11-09T21:22:41,484 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32917 is added to blk_1073741829_1005 (size=34)
2024-11-09T21:22:41,484 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:22:41,484 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-09T21:22:41,484 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:22:41,484 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:22:41,484 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-09T21:22:41,484 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:22:41,484 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:22:41,485 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731187361484Disabling compacts and flushes for region at 1731187361484Disabling writes for close at 1731187361484Writing region close event to WAL at 1731187361484Closed at 1731187361484
2024-11-09T21:22:41,485 WARN  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/.initializing
2024-11-09T21:22:41,486 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/WALs/d86ff69d6719,33537,1731187361230
2024-11-09T21:22:41,488 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C33537%2C1731187361230, suffix=, logDir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/WALs/d86ff69d6719,33537,1731187361230, archiveDir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/oldWALs, maxLogs=10
2024-11-09T21:22:41,488 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C33537%2C1731187361230.1731187361488
2024-11-09T21:22:41,493 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/WALs/d86ff69d6719,33537,1731187361230/d86ff69d6719%2C33537%2C1731187361230.1731187361488
2024-11-09T21:22:41,494 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34199:34199),(127.0.0.1/127.0.0.1:42927:42927)]
2024-11-09T21:22:41,499 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:22:41,499 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:22:41,499 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:22:41,499 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:22:41,501 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:22:41,502 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-09T21:22:41,502 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:41,503 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:41,503 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:22:41,504 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-09T21:22:41,504 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:41,505 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:22:41,505 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:22:41,506 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-09T21:22:41,506 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:41,506 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:22:41,507 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:22:41,508 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-09T21:22:41,508 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:41,509 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:22:41,509 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:22:41,510 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:22:41,510 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:22:41,511 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:22:41,511 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:22:41,512 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-09T21:22:41,513 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:22:41,515 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:22:41,516 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=871379, jitterRate=0.10801668465137482}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-09T21:22:41,516 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731187361499Initializing all the Stores at 1731187361500 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187361500Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187361500Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187361500Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187361501 (+1 ms)Cleaning up temporary data from old regions at 1731187361512 (+11 ms)Region opened successfully at 1731187361516 (+4 ms)
2024-11-09T21:22:41,517 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-09T21:22:41,521 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d22f8f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0
2024-11-09T21:22:41,522 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-09T21:22:41,522 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-09T21:22:41,522 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-09T21:22:41,522 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-09T21:22:41,523 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-09T21:22:41,523 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-09T21:22:41,523 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-09T21:22:41,525 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-09T21:22:41,526 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-09T21:22:41,540 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-09T21:22:41,541 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-09T21:22:41,541 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-09T21:22:41,549 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-09T21:22:41,549 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-09T21:22:41,550 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-09T21:22:41,557 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-09T21:22:41,559 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-09T21:22:41,565 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-09T21:22:41,567 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-09T21:22:41,574 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-09T21:22:41,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-09T21:22:41,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-09T21:22:41,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:41,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:41,583 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d86ff69d6719,33537,1731187361230, sessionid=0x10122c458830000, setting cluster-up flag (Was=false)
2024-11-09T21:22:41,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:41,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:41,624 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-09T21:22:41,625 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d86ff69d6719,33537,1731187361230
2024-11-09T21:22:41,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:41,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:41,665 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-09T21:22:41,667 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d86ff69d6719,33537,1731187361230
2024-11-09T21:22:41,668 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-09T21:22:41,670 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-09T21:22:41,670 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-09T21:22:41,670 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-09T21:22:41,670 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d86ff69d6719,33537,1731187361230
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-09T21:22:41,672 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:22:41,672 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:22:41,672 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:22:41,672 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:22:41,672 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d86ff69d6719:0, corePoolSize=10, maxPoolSize=10
2024-11-09T21:22:41,672 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:41,672 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2
2024-11-09T21:22:41,672 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:41,673 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731187391673
2024-11-09T21:22:41,673 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-09T21:22:41,673 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-09T21:22:41,673 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-09T21:22:41,673 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-09T21:22:41,673 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-09T21:22:41,673 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-09T21:22:41,674 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:41,674 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-09T21:22:41,674 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-09T21:22:41,674 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:22:41,674 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-09T21:22:41,674 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-09T21:22:41,674 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-09T21:22:41,675 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-09T21:22:41,675 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187361675,5,FailOnTimeoutGroup]
2024-11-09T21:22:41,675 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:41,675 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-09T21:22:41,678 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187361675,5,FailOnTimeoutGroup]
2024-11-09T21:22:41,678 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:41,679 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-09T21:22:41,679 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:41,679 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:41,687 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32917 is added to blk_1073741831_1007 (size=1321)
2024-11-09T21:22:41,687 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44811 is added to blk_1073741831_1007 (size=1321)
2024-11-09T21:22:41,688 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-09T21:22:41,689 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a
2024-11-09T21:22:41,690 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer(746): ClusterId : 34ba9df3-8258-4c52-a449-7d875113d2f3
2024-11-09T21:22:41,690 DEBUG [RS:0;d86ff69d6719:37671 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-09T21:22:41,699 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44811 is added to blk_1073741832_1008 (size=32)
2024-11-09T21:22:41,699 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32917 is added to blk_1073741832_1008 (size=32)
2024-11-09T21:22:41,700 DEBUG [RS:0;d86ff69d6719:37671 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-09T21:22:41,700 DEBUG [RS:0;d86ff69d6719:37671 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-09T21:22:41,700 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:22:41,701 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-09T21:22:41,703 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-09T21:22:41,703 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:41,703 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:41,704 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-09T21:22:41,705 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-09T21:22:41,705 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:41,706 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:41,706 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-09T21:22:41,707 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-09T21:22:41,707 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:41,708 DEBUG [RS:0;d86ff69d6719:37671 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-09T21:22:41,708 DEBUG [RS:0;d86ff69d6719:37671 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae0bc1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0
2024-11-09T21:22:41,708 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:41,708 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-09T21:22:41,710 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-09T21:22:41,710 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:41,711 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:41,711 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-09T21:22:41,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740
2024-11-09T21:22:41,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740
2024-11-09T21:22:41,714 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-09T21:22:41,714 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-09T21:22:41,714 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-09T21:22:41,716 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-09T21:22:41,718 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:22:41,719 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=861934, jitterRate=0.09600692987442017}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-09T21:22:41,720 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731187361700Initializing all the Stores at 1731187361701 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187361701Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187361701Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187361701Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187361701Cleaning up temporary data from old regions at 1731187361714 (+13 ms)Region opened successfully at 1731187361720 (+6 ms)
2024-11-09T21:22:41,720 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-09T21:22:41,720 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-09T21:22:41,720 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-09T21:22:41,720 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-09T21:22:41,720 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-09T21:22:41,721 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-09T21:22:41,721 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731187361720Disabling compacts and flushes for region at 1731187361720Disabling writes for close at 1731187361720Writing region close event to WAL at 1731187361721 (+1 ms)Closed at 1731187361721
2024-11-09T21:22:41,721 DEBUG [RS:0;d86ff69d6719:37671 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d86ff69d6719:37671
2024-11-09T21:22:41,721 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-09T21:22:41,721 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-09T21:22:41,721 DEBUG [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-09T21:22:41,722 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer(2659): reportForDuty to master=d86ff69d6719,33537,1731187361230 with port=37671, startcode=1731187361371
2024-11-09T21:22:41,722 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:22:41,722 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-09T21:22:41,722 DEBUG [RS:0;d86ff69d6719:37671 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-09T21:22:41,722 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-09T21:22:41,724 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-09T21:22:41,724 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35087, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-11-09T21:22:41,725 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33537 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d86ff69d6719,37671,1731187361371
2024-11-09T21:22:41,725 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33537 {}] master.ServerManager(517): Registering regionserver=d86ff69d6719,37671,1731187361371
2024-11-09T21:22:41,725 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-09T21:22:41,726 DEBUG [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a
2024-11-09T21:22:41,726 DEBUG [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42197
2024-11-09T21:22:41,726 DEBUG [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-09T21:22:41,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T21:22:41,733 DEBUG [RS:0;d86ff69d6719:37671 {}] zookeeper.ZKUtil(111): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d86ff69d6719,37671,1731187361371
2024-11-09T21:22:41,733 WARN  [RS:0;d86ff69d6719:37671 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T21:22:41,733 INFO  [RS:0;d86ff69d6719:37671 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:22:41,733 DEBUG [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371
2024-11-09T21:22:41,733 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d86ff69d6719,37671,1731187361371]
2024-11-09T21:22:41,737 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-09T21:22:41,739 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-09T21:22:41,739 INFO  [RS:0;d86ff69d6719:37671 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-09T21:22:41,739 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:41,740 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-09T21:22:41,740 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-09T21:22:41,741 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:41,741 DEBUG [RS:0;d86ff69d6719:37671 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:41,741 DEBUG [RS:0;d86ff69d6719:37671 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:41,741 DEBUG [RS:0;d86ff69d6719:37671 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:41,741 DEBUG [RS:0;d86ff69d6719:37671 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:41,741 DEBUG [RS:0;d86ff69d6719:37671 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:41,741 DEBUG [RS:0;d86ff69d6719:37671 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2
2024-11-09T21:22:41,741 DEBUG [RS:0;d86ff69d6719:37671 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:41,741 DEBUG [RS:0;d86ff69d6719:37671 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:41,741 DEBUG [RS:0;d86ff69d6719:37671 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:41,741 DEBUG [RS:0;d86ff69d6719:37671 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:41,741 DEBUG [RS:0;d86ff69d6719:37671 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:41,741 DEBUG [RS:0;d86ff69d6719:37671 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:41,741 DEBUG [RS:0;d86ff69d6719:37671 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3
2024-11-09T21:22:41,742 DEBUG [RS:0;d86ff69d6719:37671 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3
2024-11-09T21:22:41,742 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:41,742 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:41,742 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:41,742 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:41,742 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:41,742 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,37671,1731187361371-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T21:22:41,759 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-09T21:22:41,759 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,37671,1731187361371-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:41,759 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:41,759 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.Replication(171): d86ff69d6719,37671,1731187361371 started
2024-11-09T21:22:41,773 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:41,773 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer(1482): Serving as d86ff69d6719,37671,1731187361371, RpcServer on d86ff69d6719/172.17.0.2:37671, sessionid=0x10122c458830001
2024-11-09T21:22:41,773 DEBUG [RS:0;d86ff69d6719:37671 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-09T21:22:41,773 DEBUG [RS:0;d86ff69d6719:37671 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d86ff69d6719,37671,1731187361371
2024-11-09T21:22:41,773 DEBUG [RS:0;d86ff69d6719:37671 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,37671,1731187361371'
2024-11-09T21:22:41,773 DEBUG [RS:0;d86ff69d6719:37671 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-09T21:22:41,774 DEBUG [RS:0;d86ff69d6719:37671 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-09T21:22:41,774 DEBUG [RS:0;d86ff69d6719:37671 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-09T21:22:41,774 DEBUG [RS:0;d86ff69d6719:37671 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-09T21:22:41,774 DEBUG [RS:0;d86ff69d6719:37671 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d86ff69d6719,37671,1731187361371
2024-11-09T21:22:41,774 DEBUG [RS:0;d86ff69d6719:37671 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,37671,1731187361371'
2024-11-09T21:22:41,774 DEBUG [RS:0;d86ff69d6719:37671 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-09T21:22:41,775 DEBUG [RS:0;d86ff69d6719:37671 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-09T21:22:41,775 DEBUG [RS:0;d86ff69d6719:37671 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-09T21:22:41,775 INFO  [RS:0;d86ff69d6719:37671 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-09T21:22:41,775 INFO  [RS:0;d86ff69d6719:37671 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-09T21:22:41,876 WARN  [d86ff69d6719:33537 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-09T21:22:41,879 INFO  [RS:0;d86ff69d6719:37671 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C37671%2C1731187361371, suffix=, logDir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371, archiveDir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/oldWALs, maxLogs=32
2024-11-09T21:22:41,881 INFO  [RS:0;d86ff69d6719:37671 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C37671%2C1731187361371.1731187361880
2024-11-09T21:22:41,888 INFO  [RS:0;d86ff69d6719:37671 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
2024-11-09T21:22:41,889 DEBUG [RS:0;d86ff69d6719:37671 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34199:34199),(127.0.0.1/127.0.0.1:42927:42927)]
2024-11-09T21:22:42,126 DEBUG [d86ff69d6719:33537 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-09T21:22:42,127 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d86ff69d6719,37671,1731187361371
2024-11-09T21:22:42,131 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d86ff69d6719,37671,1731187361371, state=OPENING
2024-11-09T21:22:42,191 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-09T21:22:42,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:42,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:22:42,200 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-09T21:22:42,201 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:22:42,201 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:22:42,201 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d86ff69d6719,37671,1731187361371}]
2024-11-09T21:22:42,357 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-09T21:22:42,361 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36625, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-09T21:22:42,368 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-09T21:22:42,368 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:22:42,371 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C37671%2C1731187361371.meta, suffix=.meta, logDir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371, archiveDir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/oldWALs, maxLogs=32
2024-11-09T21:22:42,371 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
2024-11-09T21:22:42,377 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
2024-11-09T21:22:42,382 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42927:42927),(127.0.0.1/127.0.0.1:34199:34199)]
2024-11-09T21:22:42,387 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:22:42,387 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-09T21:22:42,387 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-09T21:22:42,387 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-09T21:22:42,387 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-09T21:22:42,387 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:22:42,387 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-09T21:22:42,387 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-09T21:22:42,389 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-09T21:22:42,390 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-09T21:22:42,390 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:42,390 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:42,390 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-09T21:22:42,391 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-09T21:22:42,391 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:42,392 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:42,392 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-09T21:22:42,392 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-09T21:22:42,392 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:42,393 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:42,393 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-09T21:22:42,394 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-09T21:22:42,394 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:42,394 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:22:42,394 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-09T21:22:42,395 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740
2024-11-09T21:22:42,396 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740
2024-11-09T21:22:42,397 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-09T21:22:42,398 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-09T21:22:42,398 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-09T21:22:42,399 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-09T21:22:42,400 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=723325, jitterRate=-0.08024550974369049}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-09T21:22:42,401 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-09T21:22:42,401 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731187362388Writing region info on filesystem at 1731187362388Initializing all the Stores at 1731187362388Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187362389 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187362389Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187362389Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187362389Cleaning up temporary data from old regions at 1731187362398 (+9 ms)Running coprocessor post-open hooks at 1731187362401 (+3 ms)Region opened successfully at 1731187362401
2024-11-09T21:22:42,402 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731187362356
2024-11-09T21:22:42,405 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-09T21:22:42,405 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-09T21:22:42,406 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,37671,1731187361371
2024-11-09T21:22:42,407 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d86ff69d6719,37671,1731187361371, state=OPEN
2024-11-09T21:22:42,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-09T21:22:42,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-09T21:22:42,451 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d86ff69d6719,37671,1731187361371
2024-11-09T21:22:42,451 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:22:42,451 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:22:42,457 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-09T21:22:42,457 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d86ff69d6719,37671,1731187361371 in 250 msec
2024-11-09T21:22:42,463 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-09T21:22:42,463 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 735 msec
2024-11-09T21:22:42,464 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:22:42,464 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-09T21:22:42,466 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-09T21:22:42,466 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,37671,1731187361371, seqNum=-1]
2024-11-09T21:22:42,466 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T21:22:42,467 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43393, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T21:22:42,474 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 804 msec
2024-11-09T21:22:42,474 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731187362474, completionTime=-1
2024-11-09T21:22:42,474 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-09T21:22:42,474 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-09T21:22:42,476 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-09T21:22:42,476 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731187422476
2024-11-09T21:22:42,476 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731187482476
2024-11-09T21:22:42,476 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-11-09T21:22:42,476 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,33537,1731187361230-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,477 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,33537,1731187361230-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,477 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,33537,1731187361230-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,477 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d86ff69d6719:33537, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,477 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,477 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,479 DEBUG [master/d86ff69d6719:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-09T21:22:42,481 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.057sec
2024-11-09T21:22:42,481 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-09T21:22:42,481 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-09T21:22:42,481 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-09T21:22:42,481 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-09T21:22:42,481 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-09T21:22:42,481 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,33537,1731187361230-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T21:22:42,481 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,33537,1731187361230-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-09T21:22:42,484 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-09T21:22:42,484 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-09T21:22:42,484 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,33537,1731187361230-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337c5dd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:22:42,491 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d86ff69d6719,33537,-1 for getting cluster id
2024-11-09T21:22:42,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-09T21:22:42,492 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '34ba9df3-8258-4c52-a449-7d875113d2f3'
2024-11-09T21:22:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-09T21:22:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "34ba9df3-8258-4c52-a449-7d875113d2f3"

2024-11-09T21:22:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39dd05c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:22:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d86ff69d6719,33537,-1]
2024-11-09T21:22:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-09T21:22:42,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:22:42,495 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-09T21:22:42,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38567dd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:22:42,496 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-09T21:22:42,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,37671,1731187361371, seqNum=-1]
2024-11-09T21:22:42,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T21:22:42,499 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T21:22:42,501 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d86ff69d6719,33537,1731187361230
2024-11-09T21:22:42,501 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:42,504 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-09T21:22:42,518 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d86ff69d6719:0 server-side Connection retries=45
2024-11-09T21:22:42,518 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:22:42,518 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T21:22:42,518 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T21:22:42,518 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:22:42,518 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T21:22:42,518 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-09T21:22:42,518 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T21:22:42,519 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40715
2024-11-09T21:22:42,520 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40715 connecting to ZooKeeper ensemble=127.0.0.1:54077
2024-11-09T21:22:42,521 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:42,522 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:22:42,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:407150x0, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T21:22:42,541 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:407150x0, quorum=127.0.0.1:54077, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-11-09T21:22:42,541 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40715-0x10122c458830002 connected
2024-11-09T21:22:42,541 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-11-09T21:22:42,542 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-09T21:22:42,543 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-09T21:22:42,543 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:40715-0x10122c458830002, quorum=127.0.0.1:54077, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-09T21:22:42,545 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40715-0x10122c458830002, quorum=127.0.0.1:54077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T21:22:42,550 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40715
2024-11-09T21:22:42,550 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40715
2024-11-09T21:22:42,550 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40715
2024-11-09T21:22:42,551 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40715
2024-11-09T21:22:42,551 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40715
2024-11-09T21:22:42,553 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.HRegionServer(746): ClusterId : 34ba9df3-8258-4c52-a449-7d875113d2f3
2024-11-09T21:22:42,553 DEBUG [RS:1;d86ff69d6719:40715 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-09T21:22:42,558 DEBUG [RS:1;d86ff69d6719:40715 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-09T21:22:42,558 DEBUG [RS:1;d86ff69d6719:40715 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-09T21:22:42,566 DEBUG [RS:1;d86ff69d6719:40715 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-09T21:22:42,567 DEBUG [RS:1;d86ff69d6719:40715 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e5512b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0
2024-11-09T21:22:42,577 DEBUG [RS:1;d86ff69d6719:40715 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;d86ff69d6719:40715
2024-11-09T21:22:42,577 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-09T21:22:42,577 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-09T21:22:42,578 DEBUG [RS:1;d86ff69d6719:40715 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-09T21:22:42,578 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.HRegionServer(2659): reportForDuty to master=d86ff69d6719,33537,1731187361230 with port=40715, startcode=1731187362517
2024-11-09T21:22:42,578 DEBUG [RS:1;d86ff69d6719:40715 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-09T21:22:42,580 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43335, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-11-09T21:22:42,580 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33537 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d86ff69d6719,40715,1731187362517
2024-11-09T21:22:42,580 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33537 {}] master.ServerManager(517): Registering regionserver=d86ff69d6719,40715,1731187362517
2024-11-09T21:22:42,582 DEBUG [RS:1;d86ff69d6719:40715 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a
2024-11-09T21:22:42,582 DEBUG [RS:1;d86ff69d6719:40715 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42197
2024-11-09T21:22:42,582 DEBUG [RS:1;d86ff69d6719:40715 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-09T21:22:42,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T21:22:42,591 DEBUG [RS:1;d86ff69d6719:40715 {}] zookeeper.ZKUtil(111): regionserver:40715-0x10122c458830002, quorum=127.0.0.1:54077, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d86ff69d6719,40715,1731187362517
2024-11-09T21:22:42,591 WARN  [RS:1;d86ff69d6719:40715 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T21:22:42,591 INFO  [RS:1;d86ff69d6719:40715 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:22:42,591 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d86ff69d6719,40715,1731187362517]
2024-11-09T21:22:42,591 DEBUG [RS:1;d86ff69d6719:40715 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517
2024-11-09T21:22:42,594 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-09T21:22:42,596 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-09T21:22:42,597 INFO  [RS:1;d86ff69d6719:40715 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-09T21:22:42,597 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,597 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-09T21:22:42,598 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-09T21:22:42,598 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,598 DEBUG [RS:1;d86ff69d6719:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:42,598 DEBUG [RS:1;d86ff69d6719:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:42,598 DEBUG [RS:1;d86ff69d6719:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:42,598 DEBUG [RS:1;d86ff69d6719:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:42,598 DEBUG [RS:1;d86ff69d6719:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:42,598 DEBUG [RS:1;d86ff69d6719:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2
2024-11-09T21:22:42,598 DEBUG [RS:1;d86ff69d6719:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:42,598 DEBUG [RS:1;d86ff69d6719:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:42,598 DEBUG [RS:1;d86ff69d6719:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:42,598 DEBUG [RS:1;d86ff69d6719:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:42,598 DEBUG [RS:1;d86ff69d6719:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:42,599 DEBUG [RS:1;d86ff69d6719:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:22:42,599 DEBUG [RS:1;d86ff69d6719:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3
2024-11-09T21:22:42,599 DEBUG [RS:1;d86ff69d6719:40715 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3
2024-11-09T21:22:42,599 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,599 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,599 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,599 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,599 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,599 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,40715,1731187362517-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T21:22:42,613 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-09T21:22:42,613 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,40715,1731187362517-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,613 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,613 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.Replication(171): d86ff69d6719,40715,1731187362517 started
2024-11-09T21:22:42,625 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:22:42,625 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.HRegionServer(1482): Serving as d86ff69d6719,40715,1731187362517, RpcServer on d86ff69d6719/172.17.0.2:40715, sessionid=0x10122c458830002
2024-11-09T21:22:42,625 DEBUG [RS:1;d86ff69d6719:40715 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-09T21:22:42,625 DEBUG [RS:1;d86ff69d6719:40715 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d86ff69d6719,40715,1731187362517
2024-11-09T21:22:42,625 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;d86ff69d6719:40715,5,FailOnTimeoutGroup]
2024-11-09T21:22:42,625 DEBUG [RS:1;d86ff69d6719:40715 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,40715,1731187362517'
2024-11-09T21:22:42,625 DEBUG [RS:1;d86ff69d6719:40715 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-09T21:22:42,626 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-11-09T21:22:42,626 DEBUG [RS:1;d86ff69d6719:40715 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-09T21:22:42,626 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-09T21:22:42,626 DEBUG [RS:1;d86ff69d6719:40715 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-09T21:22:42,626 DEBUG [RS:1;d86ff69d6719:40715 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-09T21:22:42,627 DEBUG [RS:1;d86ff69d6719:40715 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d86ff69d6719,40715,1731187362517
2024-11-09T21:22:42,627 DEBUG [RS:1;d86ff69d6719:40715 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,40715,1731187362517'
2024-11-09T21:22:42,627 DEBUG [RS:1;d86ff69d6719:40715 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-09T21:22:42,627 DEBUG [RS:1;d86ff69d6719:40715 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-09T21:22:42,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is d86ff69d6719,33537,1731187361230
2024-11-09T21:22:42,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3deaf92c
2024-11-09T21:22:42,627 DEBUG [RS:1;d86ff69d6719:40715 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-09T21:22:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-09T21:22:42,628 INFO  [RS:1;d86ff69d6719:40715 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-09T21:22:42,628 INFO  [RS:1;d86ff69d6719:40715 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-09T21:22:42,629 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-09T21:22:42,630 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33537 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-09T21:22:42,630 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33537 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-09T21:22:42,630 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33537 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-09T21:22:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33537 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-11-09T21:22:42,632 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-11-09T21:22:42,633 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:42,633 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33537 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-11-09T21:22:42,634 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-09T21:22:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-09T21:22:42,642 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32917 is added to blk_1073741835_1011 (size=393)
2024-11-09T21:22:42,642 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44811 is added to blk_1073741835_1011 (size=393)
2024-11-09T21:22:42,644 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 922e61fda0bc2cc7cecfa696c94b4f30, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a
2024-11-09T21:22:42,651 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44811 is added to blk_1073741836_1012 (size=76)
2024-11-09T21:22:42,651 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32917 is added to blk_1073741836_1012 (size=76)
2024-11-09T21:22:42,652 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:22:42,652 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 922e61fda0bc2cc7cecfa696c94b4f30, disabling compactions & flushes
2024-11-09T21:22:42,652 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.
2024-11-09T21:22:42,652 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.
2024-11-09T21:22:42,652 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30. after waiting 0 ms
2024-11-09T21:22:42,652 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.
2024-11-09T21:22:42,652 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.
2024-11-09T21:22:42,652 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 922e61fda0bc2cc7cecfa696c94b4f30:
Waiting for close lock at 1731187362652Disabling compacts and flushes for region at 1731187362652Disabling writes for close at 1731187362652Writing region close event to WAL at 1731187362652Closed at 1731187362652
2024-11-09T21:22:42,654 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-11-09T21:22:42,655 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731187362654"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731187362654"}]},"ts":"1731187362654"}
2024-11-09T21:22:42,657 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-09T21:22:42,658 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-09T21:22:42,659 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187362659"}]},"ts":"1731187362659"}
2024-11-09T21:22:42,661 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-11-09T21:22:42,661 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=922e61fda0bc2cc7cecfa696c94b4f30, ASSIGN}]
2024-11-09T21:22:42,663 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=922e61fda0bc2cc7cecfa696c94b4f30, ASSIGN
2024-11-09T21:22:42,664 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=922e61fda0bc2cc7cecfa696c94b4f30, ASSIGN; state=OFFLINE, location=d86ff69d6719,37671,1731187361371; forceNewPlan=false, retain=false
2024-11-09T21:22:42,733 INFO  [RS:1;d86ff69d6719:40715 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C40715%2C1731187362517, suffix=, logDir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517, archiveDir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/oldWALs, maxLogs=32
2024-11-09T21:22:42,735 INFO  [RS:1;d86ff69d6719:40715 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C40715%2C1731187362517.1731187362735
2024-11-09T21:22:42,743 INFO  [RS:1;d86ff69d6719:40715 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
2024-11-09T21:22:42,746 DEBUG [RS:1;d86ff69d6719:40715 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34199:34199),(127.0.0.1/127.0.0.1:42927:42927)]
2024-11-09T21:22:42,815 INFO  [d86ff69d6719:33537 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-11-09T21:22:42,816 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=922e61fda0bc2cc7cecfa696c94b4f30, regionState=OPENING, regionLocation=d86ff69d6719,37671,1731187361371
2024-11-09T21:22:42,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=922e61fda0bc2cc7cecfa696c94b4f30, ASSIGN because future has completed
2024-11-09T21:22:42,819 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 922e61fda0bc2cc7cecfa696c94b4f30, server=d86ff69d6719,37671,1731187361371}]
2024-11-09T21:22:42,984 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.
2024-11-09T21:22:42,984 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 922e61fda0bc2cc7cecfa696c94b4f30, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:22:42,984 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:22:42,985 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:22:42,985 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:22:42,985 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:22:42,987 INFO  [StoreOpener-922e61fda0bc2cc7cecfa696c94b4f30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 922e61fda0bc2cc7cecfa696c94b4f30 
2024-11-09T21:22:42,988 INFO  [StoreOpener-922e61fda0bc2cc7cecfa696c94b4f30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 922e61fda0bc2cc7cecfa696c94b4f30 columnFamilyName info
2024-11-09T21:22:42,989 DEBUG [StoreOpener-922e61fda0bc2cc7cecfa696c94b4f30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:22:42,989 INFO  [StoreOpener-922e61fda0bc2cc7cecfa696c94b4f30-1 {}] regionserver.HStore(327): Store=922e61fda0bc2cc7cecfa696c94b4f30/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:22:42,989 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:22:42,990 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:22:42,991 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:22:42,992 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:22:42,992 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:22:42,994 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:22:42,996 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:22:42,997 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 922e61fda0bc2cc7cecfa696c94b4f30; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=868761, jitterRate=0.10468696057796478}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-09T21:22:42,997 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:22:42,997 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 922e61fda0bc2cc7cecfa696c94b4f30:
Running coprocessor pre-open hook at 1731187362985Writing region info on filesystem at 1731187362985Initializing all the Stores at 1731187362986 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187362986Cleaning up temporary data from old regions at 1731187362992 (+6 ms)Running coprocessor post-open hooks at 1731187362997 (+5 ms)Region opened successfully at 1731187362997
2024-11-09T21:22:42,999 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30., pid=6, masterSystemTime=1731187362973
2024-11-09T21:22:43,001 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.
2024-11-09T21:22:43,001 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.
2024-11-09T21:22:43,002 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=922e61fda0bc2cc7cecfa696c94b4f30, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,37671,1731187361371
2024-11-09T21:22:43,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 922e61fda0bc2cc7cecfa696c94b4f30, server=d86ff69d6719,37671,1731187361371 because future has completed
2024-11-09T21:22:43,010 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-09T21:22:43,010 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 922e61fda0bc2cc7cecfa696c94b4f30, server=d86ff69d6719,37671,1731187361371 in 187 msec
2024-11-09T21:22:43,014 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-09T21:22:43,014 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=922e61fda0bc2cc7cecfa696c94b4f30, ASSIGN in 349 msec
2024-11-09T21:22:43,015 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-09T21:22:43,015 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187363015"}]},"ts":"1731187363015"}
2024-11-09T21:22:43,018 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-11-09T21:22:43,020 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-11-09T21:22:43,023 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 390 msec
2024-11-09T21:22:47,893 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-09T21:22:47,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:22:47,916 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:22:47,918 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:22:47,918 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:22:47,925 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-11-09T21:22:49,563 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-09T21:22:49,564 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-09T21:22:49,567 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-09T21:22:49,567 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-11-09T21:22:49,568 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-09T21:22:49,569 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-09T21:22:49,569 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-09T21:22:49,569 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-09T21:22:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33537 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-09T21:22:52,694 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-11-09T21:22:52,694 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-11-09T21:22:52,698 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-11-09T21:22:52,698 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.
2024-11-09T21:22:52,713 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:22:52,716 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:22:52,717 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:22:52,717 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:22:52,717 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T21:22:52,717 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ca82099{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:22:52,718 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ed35b1e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:22:52,810 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1196c8fc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/java.io.tmpdir/jetty-localhost-37671-hadoop-hdfs-3_4_1-tests_jar-_-any-10677709469583703453/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:22:52,811 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5580c33e{HTTP/1.1, (http/1.1)}{localhost:37671}
2024-11-09T21:22:52,811 INFO  [Time-limited test {}] server.Server(415): Started @120230ms
2024-11-09T21:22:52,812 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:22:52,844 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:22:52,847 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:22:52,848 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:22:52,848 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:22:52,848 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-09T21:22:52,849 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fe58b15{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:22:52,849 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5455501c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:22:52,943 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5e08dd81{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/java.io.tmpdir/jetty-localhost-35967-hadoop-hdfs-3_4_1-tests_jar-_-any-7607854886439965701/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:22:52,943 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@178f342a{HTTP/1.1, (http/1.1)}{localhost:35967}
2024-11-09T21:22:52,943 INFO  [Time-limited test {}] server.Server(415): Started @120362ms
2024-11-09T21:22:52,945 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:22:52,976 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:22:52,979 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:22:52,980 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:22:52,980 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:22:52,980 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-09T21:22:52,981 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@792fa80c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:22:52,981 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d4ec789{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:22:53,074 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5dd0b56c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/java.io.tmpdir/jetty-localhost-35073-hadoop-hdfs-3_4_1-tests_jar-_-any-16408677865821131738/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:22:53,074 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3740407e{HTTP/1.1, (http/1.1)}{localhost:35073}
2024-11-09T21:22:53,074 INFO  [Time-limited test {}] server.Server(415): Started @120493ms
2024-11-09T21:22:53,075 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:22:53,682 WARN  [Thread-866 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data5/current/BP-1331531764-172.17.0.2-1731187359371/current, will proceed with Du for space computation calculation, 
2024-11-09T21:22:53,683 WARN  [Thread-867 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data6/current/BP-1331531764-172.17.0.2-1731187359371/current, will proceed with Du for space computation calculation, 
2024-11-09T21:22:53,703 WARN  [Thread-808 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:22:53,705 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9748f85593be3699 with lease ID 0xfd543dc177bc545c: Processing first storage report for DS-f772052f-7fcd-4837-b069-59cfbccaba2b from datanode DatanodeRegistration(127.0.0.1:40849, datanodeUuid=aefa749f-3c94-4908-8f6e-fc4fa87a2ba1, infoPort=46631, infoSecurePort=0, ipcPort=36185, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371)
2024-11-09T21:22:53,705 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9748f85593be3699 with lease ID 0xfd543dc177bc545c: from storage DS-f772052f-7fcd-4837-b069-59cfbccaba2b node DatanodeRegistration(127.0.0.1:40849, datanodeUuid=aefa749f-3c94-4908-8f6e-fc4fa87a2ba1, infoPort=46631, infoSecurePort=0, ipcPort=36185, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:22:53,706 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9748f85593be3699 with lease ID 0xfd543dc177bc545c: Processing first storage report for DS-a5d8ab21-b0b2-4c30-ad48-6d8dfad3e2e3 from datanode DatanodeRegistration(127.0.0.1:40849, datanodeUuid=aefa749f-3c94-4908-8f6e-fc4fa87a2ba1, infoPort=46631, infoSecurePort=0, ipcPort=36185, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371)
2024-11-09T21:22:53,706 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9748f85593be3699 with lease ID 0xfd543dc177bc545c: from storage DS-a5d8ab21-b0b2-4c30-ad48-6d8dfad3e2e3 node DatanodeRegistration(127.0.0.1:40849, datanodeUuid=aefa749f-3c94-4908-8f6e-fc4fa87a2ba1, infoPort=46631, infoSecurePort=0, ipcPort=36185, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:22:53,905 WARN  [Thread-878 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data7/current/BP-1331531764-172.17.0.2-1731187359371/current, will proceed with Du for space computation calculation, 
2024-11-09T21:22:53,905 WARN  [Thread-879 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data8/current/BP-1331531764-172.17.0.2-1731187359371/current, will proceed with Du for space computation calculation, 
2024-11-09T21:22:53,925 WARN  [Thread-830 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:22:53,927 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf07ed92d697522fb with lease ID 0xfd543dc177bc545d: Processing first storage report for DS-68d83e8b-f808-47df-88af-73a926e3265c from datanode DatanodeRegistration(127.0.0.1:34361, datanodeUuid=141d9b99-a758-4387-9668-071f4ea5ccf7, infoPort=37607, infoSecurePort=0, ipcPort=39687, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371)
2024-11-09T21:22:53,927 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf07ed92d697522fb with lease ID 0xfd543dc177bc545d: from storage DS-68d83e8b-f808-47df-88af-73a926e3265c node DatanodeRegistration(127.0.0.1:34361, datanodeUuid=141d9b99-a758-4387-9668-071f4ea5ccf7, infoPort=37607, infoSecurePort=0, ipcPort=39687, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:22:53,927 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf07ed92d697522fb with lease ID 0xfd543dc177bc545d: Processing first storage report for DS-580f6b9f-ce39-4a87-b61a-32f623cf9644 from datanode DatanodeRegistration(127.0.0.1:34361, datanodeUuid=141d9b99-a758-4387-9668-071f4ea5ccf7, infoPort=37607, infoSecurePort=0, ipcPort=39687, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371)
2024-11-09T21:22:53,927 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf07ed92d697522fb with lease ID 0xfd543dc177bc545d: from storage DS-580f6b9f-ce39-4a87-b61a-32f623cf9644 node DatanodeRegistration(127.0.0.1:34361, datanodeUuid=141d9b99-a758-4387-9668-071f4ea5ccf7, infoPort=37607, infoSecurePort=0, ipcPort=39687, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:22:54,012 WARN  [Thread-889 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data9/current/BP-1331531764-172.17.0.2-1731187359371/current, will proceed with Du for space computation calculation, 
2024-11-09T21:22:54,012 WARN  [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data10/current/BP-1331531764-172.17.0.2-1731187359371/current, will proceed with Du for space computation calculation, 
2024-11-09T21:22:54,035 WARN  [Thread-852 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:22:54,037 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xec0ba43bd845c3e9 with lease ID 0xfd543dc177bc545e: Processing first storage report for DS-026c0227-9d92-4566-9f41-fc21b2c4222d from datanode DatanodeRegistration(127.0.0.1:36141, datanodeUuid=d14ffb12-89ea-49c8-a984-427fb381fc9e, infoPort=39337, infoSecurePort=0, ipcPort=34833, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371)
2024-11-09T21:22:54,037 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xec0ba43bd845c3e9 with lease ID 0xfd543dc177bc545e: from storage DS-026c0227-9d92-4566-9f41-fc21b2c4222d node DatanodeRegistration(127.0.0.1:36141, datanodeUuid=d14ffb12-89ea-49c8-a984-427fb381fc9e, infoPort=39337, infoSecurePort=0, ipcPort=34833, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:22:54,037 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xec0ba43bd845c3e9 with lease ID 0xfd543dc177bc545e: Processing first storage report for DS-876f3b3f-a8d3-4ce8-8ae4-51a3b4aa3c48 from datanode DatanodeRegistration(127.0.0.1:36141, datanodeUuid=d14ffb12-89ea-49c8-a984-427fb381fc9e, infoPort=39337, infoSecurePort=0, ipcPort=34833, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371)
2024-11-09T21:22:54,037 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xec0ba43bd845c3e9 with lease ID 0xfd543dc177bc545e: from storage DS-876f3b3f-a8d3-4ce8-8ae4-51a3b4aa3c48 node DatanodeRegistration(127.0.0.1:36141, datanodeUuid=d14ffb12-89ea-49c8-a984-427fb381fc9e, infoPort=39337, infoSecurePort=0, ipcPort=34833, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:22:54,104 WARN  [ResponseProcessor for block BP-1331531764-172.17.0.2-1731187359371:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1331531764-172.17.0.2-1731187359371:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-1331531764-172.17.0.2-1731187359371:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:54,104 WARN  [ResponseProcessor for block BP-1331531764-172.17.0.2-1731187359371:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1331531764-172.17.0.2-1731187359371:blk_1073741837_1013
java.io.IOException: Bad response ERROR for BP-1331531764-172.17.0.2-1731187359371:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:54,103 WARN  [ResponseProcessor for block BP-1331531764-172.17.0.2-1731187359371:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1331531764-172.17.0.2-1731187359371:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:54,104 WARN  [ResponseProcessor for block BP-1331531764-172.17.0.2-1731187359371:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1331531764-172.17.0.2-1731187359371:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-1331531764-172.17.0.2-1731187359371:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:54,105 WARN  [DataStreamer for file /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735 block BP-1331531764-172.17.0.2-1731187359371:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK], DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]) is bad.
2024-11-09T21:22:54,105 WARN  [DataStreamer for file /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta block BP-1331531764-172.17.0.2-1731187359371:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK], DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]) is bad.
2024-11-09T21:22:54,105 WARN  [DataStreamer for file /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880 block BP-1331531764-172.17.0.2-1731187359371:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK], DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]) is bad.
2024-11-09T21:22:54,105 WARN  [DataStreamer for file /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/WALs/d86ff69d6719,33537,1731187361230/d86ff69d6719%2C33537%2C1731187361230.1731187361488 block BP-1331531764-172.17.0.2-1731187359371:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK], DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]) is bad.
2024-11-09T21:22:54,105 WARN  [PacketResponder: BP-1331531764-172.17.0.2-1731187359371:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:44811] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:22:54,106 WARN  [PacketResponder: BP-1331531764-172.17.0.2-1731187359371:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:44811] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:22:54,106 WARN  [PacketResponder: BP-1331531764-172.17.0.2-1731187359371:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:44811] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:22:54,108 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:58442 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:32917:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58442 dst: /127.0.0.1:32917
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:22:54,108 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1411a57a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:22:54,108 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_879490670_22 at /127.0.0.1:58474 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:32917:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58474 dst: /127.0.0.1:32917
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:22:54,108 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54950 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:44811:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54950 dst: /127.0.0.1:44811
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:22:54,107 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54958 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:44811:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54958 dst: /127.0.0.1:44811
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:22:54,109 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1341187650_22 at /127.0.0.1:52142 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:32917:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52142 dst: /127.0.0.1:32917
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:22:54,109 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:58446 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:32917:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58446 dst: /127.0.0.1:32917
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:22:54,109 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4495a8b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:22:54,109 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:22:54,109 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_879490670_22 at /127.0.0.1:54986 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:44811:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54986 dst: /127.0.0.1:44811
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:22:54,110 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7891c14c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:22:54,110 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@73b17c7e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.log.dir/,STOPPED}
2024-11-09T21:22:54,109 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1341187650_22 at /127.0.0.1:53272 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:44811:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53272 dst: /127.0.0.1:44811
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:22:54,112 WARN  [BP-1331531764-172.17.0.2-1731187359371 heartbeating to localhost/127.0.0.1:42197 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:22:54,112 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:22:54,112 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:22:54,112 WARN  [BP-1331531764-172.17.0.2-1731187359371 heartbeating to localhost/127.0.0.1:42197 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1331531764-172.17.0.2-1731187359371 (Datanode Uuid 2894e6a3-288e-408f-8bd5-b84a909e5593) service to localhost/127.0.0.1:42197
2024-11-09T21:22:54,112 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data3/current/BP-1331531764-172.17.0.2-1731187359371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:22:54,113 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data4/current/BP-1331531764-172.17.0.2-1731187359371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:22:54,113 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:22:54,113 WARN  [DataStreamer for file /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta block BP-1331531764-172.17.0.2-1731187359371:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:54,113 WARN  [DataStreamer for file /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880 block BP-1331531764-172.17.0.2-1731187359371:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:54,113 WARN  [DataStreamer for file /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/WALs/d86ff69d6719,33537,1731187361230/d86ff69d6719%2C33537%2C1731187361230.1731187361488 block BP-1331531764-172.17.0.2-1731187359371:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Sender.send(Sender.java:84) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Sender.writeBlock(Sender.java:171) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1922) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:54,114 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@791c8af9 {}] datanode.DataXceiver(331): 127.0.0.1:32917:DataXceiver error processing unknown operation  src: /127.0.0.1:38614 dst: /127.0.0.1:32917
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:22:54,114 WARN  [DataStreamer for file /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735 block BP-1331531764-172.17.0.2-1731187359371:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:54,117 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7d85f4da{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:22:54,117 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@32d6576d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:22:54,117 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:22:54,117 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@471af73e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:22:54,117 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d4a4bdf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.log.dir/,STOPPED}
2024-11-09T21:22:54,118 WARN  [BP-1331531764-172.17.0.2-1731187359371 heartbeating to localhost/127.0.0.1:42197 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:22:54,118 WARN  [BP-1331531764-172.17.0.2-1731187359371 heartbeating to localhost/127.0.0.1:42197 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1331531764-172.17.0.2-1731187359371 (Datanode Uuid ac5cfbe0-8e00-4177-b0da-f34af1b9eff1) service to localhost/127.0.0.1:42197
2024-11-09T21:22:54,118 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:22:54,119 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:22:54,119 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data1/current/BP-1331531764-172.17.0.2-1731187359371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:22:54,119 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data2/current/BP-1331531764-172.17.0.2-1731187359371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:22:54,119 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:22:54,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30., hostname=d86ff69d6719,37671,1731187361371, seqNum=2]
2024-11-09T21:22:54,124 ERROR [FSHLog-0-hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a-prefix:d86ff69d6719,37671,1731187361371 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:54,124 WARN  [FSHLog-0-hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a-prefix:d86ff69d6719,37671,1731187361371 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:54,124 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d86ff69d6719%2C37671%2C1731187361371:(num 1731187361880) roll requested
2024-11-09T21:22:54,124 INFO  [regionserver/d86ff69d6719:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C37671%2C1731187361371.1731187374124
2024-11-09T21:22:54,131 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:54,131 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:54,131 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:54,131 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:54,131 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:54,131 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187374124
2024-11-09T21:22:54,132 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:54,133 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:54,133 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46631:46631),(127.0.0.1/127.0.0.1:39337:39337)]
2024-11-09T21:22:54,133 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880 is not closed yet, will try archiving it next time
2024-11-09T21:22:54,134 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-11-09T21:22:54,134 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-11-09T21:22:54,134 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
2024-11-09T21:22:54,136 WARN  [IPC Server handler 1 on default port 42197 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1009
2024-11-09T21:22:54,139 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880 after 3ms
2024-11-09T21:22:54,600 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:55,469 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:56,133 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:56,135 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187374124
2024-11-09T21:22:56,136 WARN  [ResponseProcessor for block BP-1331531764-172.17.0.2-1731187359371:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1331531764-172.17.0.2-1731187359371:blk_1073741838_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:56,137 WARN  [DataStreamer for file /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187374124 block BP-1331531764-172.17.0.2-1731187359371:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK], DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]) is bad.
2024-11-09T21:22:56,138 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:36164 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:40849:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36164 dst: /127.0.0.1:40849
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:22:56,139 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:42928 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:36141:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42928 dst: /127.0.0.1:36141
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:22:56,173 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1196c8fc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:22:56,174 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5580c33e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:22:56,174 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:22:56,174 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ed35b1e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:22:56,174 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ca82099{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.log.dir/,STOPPED}
2024-11-09T21:22:56,175 WARN  [BP-1331531764-172.17.0.2-1731187359371 heartbeating to localhost/127.0.0.1:42197 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:22:56,175 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:22:56,175 WARN  [BP-1331531764-172.17.0.2-1731187359371 heartbeating to localhost/127.0.0.1:42197 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1331531764-172.17.0.2-1731187359371 (Datanode Uuid aefa749f-3c94-4908-8f6e-fc4fa87a2ba1) service to localhost/127.0.0.1:42197
2024-11-09T21:22:56,175 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:22:56,176 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data5/current/BP-1331531764-172.17.0.2-1731187359371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:22:56,176 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data6/current/BP-1331531764-172.17.0.2-1731187359371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:22:56,176 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:22:56,601 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:57,469 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:58,135 WARN  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]]
2024-11-09T21:22:58,135 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:58,135 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d86ff69d6719%2C37671%2C1731187361371:(num 1731187374124) roll requested
2024-11-09T21:22:58,136 INFO  [regionserver/d86ff69d6719:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C37671%2C1731187361371.1731187378136
2024-11-09T21:22:58,141 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880 after 4007ms
2024-11-09T21:22:58,144 WARN  [Thread-910 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1021
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:58,145 WARN  [Thread-910 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK], DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]) is bad.
2024-11-09T21:22:58,145 WARN  [Thread-910 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741839_1021
2024-11-09T21:22:58,149 WARN  [Thread-910 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]
2024-11-09T21:22:58,156 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:58,156 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:58,156 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:58,156 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:58,156 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:22:58,156 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187374124 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187378136
2024-11-09T21:22:58,157 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37607:37607),(127.0.0.1/127.0.0.1:39337:39337)]
2024-11-09T21:22:58,157 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880 is not closed yet, will try archiving it next time
2024-11-09T21:22:58,158 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187374124 is not closed yet, will try archiving it next time
2024-11-09T21:22:58,159 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36141 is added to blk_1073741838_1020 (size=2431)
2024-11-09T21:22:58,181 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-09T21:22:58,560 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880 is not closed yet, will try archiving it next time
2024-11-09T21:22:58,602 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:22:59,470 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:00,158 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:00,187 WARN  [ResponseProcessor for block BP-1331531764-172.17.0.2-1731187359371:blk_1073741840_1022 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1331531764-172.17.0.2-1731187359371:blk_1073741840_1022
java.io.IOException: Bad response ERROR for BP-1331531764-172.17.0.2-1731187359371:blk_1073741840_1022 from datanode DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:00,188 WARN  [DataStreamer for file /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187378136 block BP-1331531764-172.17.0.2-1731187359371:blk_1073741840_1022 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK], DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]) is bad.
2024-11-09T21:23:00,188 WARN  [PacketResponder: BP-1331531764-172.17.0.2-1731187359371:blk_1073741840_1022, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:36141] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:00,190 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:38136 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:34361:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38136 dst: /127.0.0.1:34361
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:00,190 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:42938 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:36141:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42938 dst: /127.0.0.1:36141
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:00,198 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5dd0b56c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:23:00,198 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3740407e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:23:00,198 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:23:00,198 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d4ec789{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:23:00,198 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@792fa80c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.log.dir/,STOPPED}
2024-11-09T21:23:00,200 WARN  [BP-1331531764-172.17.0.2-1731187359371 heartbeating to localhost/127.0.0.1:42197 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:23:00,200 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:23:00,200 WARN  [BP-1331531764-172.17.0.2-1731187359371 heartbeating to localhost/127.0.0.1:42197 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1331531764-172.17.0.2-1731187359371 (Datanode Uuid d14ffb12-89ea-49c8-a984-427fb381fc9e) service to localhost/127.0.0.1:42197
2024-11-09T21:23:00,200 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:23:00,200 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data9/current/BP-1331531764-172.17.0.2-1731187359371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:23:00,201 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data10/current/BP-1331531764-172.17.0.2-1731187359371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:23:00,201 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:23:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37671 {}] regionserver.HRegion(8855): Flush requested on 922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:23:00,210 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 922e61fda0bc2cc7cecfa696c94b4f30 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-09T21:23:00,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/748681af973648a89d55225e157d5749 is 1080, key is row0002/info:/1731187376177/Put/seqid=0
2024-11-09T21:23:00,234 WARN  [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1024
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:00,235 WARN  [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741841_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK], DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]) is bad.
2024-11-09T21:23:00,235 WARN  [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741841_1024
2024-11-09T21:23:00,235 WARN  [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]
2024-11-09T21:23:00,236 WARN  [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:00,236 WARN  [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK], DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]) is bad.
2024-11-09T21:23:00,236 WARN  [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741842_1025
2024-11-09T21:23:00,237 WARN  [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]
2024-11-09T21:23:00,239 WARN  [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40849
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:00,239 WARN  [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK], DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]) is bad.
2024-11-09T21:23:00,239 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:38152 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741843_1026] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data8]'}, localName='127.0.0.1:34361', datanodeUuid='141d9b99-a758-4387-9668-071f4ea5ccf7', xmitsInProgress=0}:Exception transferring block BP-1331531764-172.17.0.2-1731187359371:blk_1073741843_1026 to mirror 127.0.0.1:40849
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:00,239 WARN  [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741843_1026
2024-11-09T21:23:00,239 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:38152 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741843_1026] {}] datanode.BlockReceiver(316): Block 1073741843 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-09T21:23:00,239 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:38152 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741843_1026] {}] datanode.DataXceiver(331): 127.0.0.1:34361:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38152 dst: /127.0.0.1:34361
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:00,240 WARN  [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]
2024-11-09T21:23:00,241 WARN  [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:00,241 WARN  [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK], DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]) is bad.
2024-11-09T21:23:00,241 WARN  [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741844_1027
2024-11-09T21:23:00,242 WARN  [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]
2024-11-09T21:23:00,242 WARN  [IPC Server handler 2 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-09T21:23:00,242 WARN  [IPC Server handler 2 on default port 42197 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-09T21:23:00,242 WARN  [IPC Server handler 2 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-09T21:23:00,246 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741845_1028 (size=10347)
2024-11-09T21:23:00,602 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:00,647 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/748681af973648a89d55225e157d5749
2024-11-09T21:23:00,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/748681af973648a89d55225e157d5749 as hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/748681af973648a89d55225e157d5749
2024-11-09T21:23:00,667 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/748681af973648a89d55225e157d5749, entries=5, sequenceid=11, filesize=10.1 K
2024-11-09T21:23:00,668 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 922e61fda0bc2cc7cecfa696c94b4f30 in 458ms, sequenceid=11, compaction requested=false
2024-11-09T21:23:00,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 922e61fda0bc2cc7cecfa696c94b4f30:

2024-11-09T21:23:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37671 {}] regionserver.HRegion(8855): Flush requested on 922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:23:00,845 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 922e61fda0bc2cc7cecfa696c94b4f30 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-11-09T21:23:00,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/78c276bdc48b41aa8b5f2ac16ce712e9 is 1080, key is row0007/info:/1731187380212/Put/seqid=0
2024-11-09T21:23:00,856 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:00,857 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK], DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]) is bad.
2024-11-09T21:23:00,857 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741846_1029
2024-11-09T21:23:00,859 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]
2024-11-09T21:23:00,860 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:32917
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:00,860 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:38184 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741847_1030] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data8]'}, localName='127.0.0.1:34361', datanodeUuid='141d9b99-a758-4387-9668-071f4ea5ccf7', xmitsInProgress=0}:Exception transferring block BP-1331531764-172.17.0.2-1731187359371:blk_1073741847_1030 to mirror 127.0.0.1:32917
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:00,861 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK], DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]) is bad.
2024-11-09T21:23:00,861 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741847_1030
2024-11-09T21:23:00,861 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:38184 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741847_1030] {}] datanode.BlockReceiver(316): Block 1073741847 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-09T21:23:00,861 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:38184 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741847_1030] {}] datanode.DataXceiver(331): 127.0.0.1:34361:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38184 dst: /127.0.0.1:34361
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:00,861 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]
2024-11-09T21:23:00,862 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:00,863 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK], DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]) is bad.
2024-11-09T21:23:00,863 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741848_1031
2024-11-09T21:23:00,863 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]
2024-11-09T21:23:00,865 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44811
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:00,865 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:38194 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741849_1032] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data8]'}, localName='127.0.0.1:34361', datanodeUuid='141d9b99-a758-4387-9668-071f4ea5ccf7', xmitsInProgress=0}:Exception transferring block BP-1331531764-172.17.0.2-1731187359371:blk_1073741849_1032 to mirror 127.0.0.1:44811
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:00,865 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK], DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]) is bad.
2024-11-09T21:23:00,865 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741849_1032
2024-11-09T21:23:00,865 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:38194 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741849_1032] {}] datanode.BlockReceiver(316): Block 1073741849 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-09T21:23:00,865 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:38194 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741849_1032] {}] datanode.DataXceiver(331): 127.0.0.1:34361:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38194 dst: /127.0.0.1:34361
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:00,866 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]
2024-11-09T21:23:00,866 WARN  [IPC Server handler 2 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-09T21:23:00,866 WARN  [IPC Server handler 2 on default port 42197 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-09T21:23:00,866 WARN  [IPC Server handler 2 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-09T21:23:00,869 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741850_1033 (size=12506)
2024-11-09T21:23:01,270 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/78c276bdc48b41aa8b5f2ac16ce712e9
2024-11-09T21:23:01,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/78c276bdc48b41aa8b5f2ac16ce712e9 as hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/78c276bdc48b41aa8b5f2ac16ce712e9
2024-11-09T21:23:01,285 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/78c276bdc48b41aa8b5f2ac16ce712e9, entries=7, sequenceid=24, filesize=12.2 K
2024-11-09T21:23:01,286 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for 922e61fda0bc2cc7cecfa696c94b4f30 in 442ms, sequenceid=24, compaction requested=false
2024-11-09T21:23:01,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 922e61fda0bc2cc7cecfa696c94b4f30:

2024-11-09T21:23:01,286 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-11-09T21:23:01,286 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:23:01,286 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/78c276bdc48b41aa8b5f2ac16ce712e9 because midkey is the same as first or last row
2024-11-09T21:23:01,470 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:01,948 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@65249791[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:34361, datanodeUuid=141d9b99-a758-4387-9668-071f4ea5ccf7, infoPort=37607, infoSecurePort=0, ipcPort=39687, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371):Failed to transfer BP-1331531764-172.17.0.2-1731187359371:blk_1073741850_1033 to 127.0.0.1:32917 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:01,948 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@14cfbac1[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:34361, datanodeUuid=141d9b99-a758-4387-9668-071f4ea5ccf7, infoPort=37607, infoSecurePort=0, ipcPort=39687, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371):Failed to transfer BP-1331531764-172.17.0.2-1731187359371:blk_1073741845_1028 to 127.0.0.1:36141 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:02,159 WARN  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK]]
2024-11-09T21:23:02,159 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:02,159 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d86ff69d6719%2C37671%2C1731187361371:(num 1731187378136) roll requested
2024-11-09T21:23:02,161 INFO  [regionserver/d86ff69d6719:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C37671%2C1731187361371.1731187382159
2024-11-09T21:23:02,165 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:02,165 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK], DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]) is bad.
2024-11-09T21:23:02,166 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741851_1034
2024-11-09T21:23:02,166 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]
2024-11-09T21:23:02,168 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:02,169 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK], DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]) is bad.
2024-11-09T21:23:02,169 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741852_1035
2024-11-09T21:23:02,170 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]
2024-11-09T21:23:02,171 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:02,172 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK], DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]) is bad.
2024-11-09T21:23:02,172 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741853_1036
2024-11-09T21:23:02,172 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]
2024-11-09T21:23:02,175 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44811
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:02,175 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54530 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741854_1037] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data8]'}, localName='127.0.0.1:34361', datanodeUuid='141d9b99-a758-4387-9668-071f4ea5ccf7', xmitsInProgress=0}:Exception transferring block BP-1331531764-172.17.0.2-1731187359371:blk_1073741854_1037 to mirror 127.0.0.1:44811
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:02,176 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK], DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]) is bad.
2024-11-09T21:23:02,176 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54530 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741854_1037] {}] datanode.BlockReceiver(316): Block 1073741854 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-09T21:23:02,176 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741854_1037
2024-11-09T21:23:02,176 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54530 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741854_1037] {}] datanode.DataXceiver(331): 127.0.0.1:34361:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54530 dst: /127.0.0.1:34361
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:02,177 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]
2024-11-09T21:23:02,178 WARN  [IPC Server handler 3 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-09T21:23:02,178 WARN  [IPC Server handler 3 on default port 42197 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-09T21:23:02,178 WARN  [IPC Server handler 3 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-09T21:23:02,181 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:02,181 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:02,181 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:02,181 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:02,181 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:02,182 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187378136 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187382159
2024-11-09T21:23:02,182 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37607:37607)]
2024-11-09T21:23:02,182 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880 is not closed yet, will try archiving it next time
2024-11-09T21:23:02,183 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187378136 is not closed yet, will try archiving it next time
2024-11-09T21:23:02,183 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187374124 to hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/oldWALs/d86ff69d6719%2C37671%2C1731187361371.1731187374124
2024-11-09T21:23:02,183 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741840_1023 (size=25992)
2024-11-09T21:23:02,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37671 {}] regionserver.HRegion(8855): Flush requested on 922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:23:02,287 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 922e61fda0bc2cc7cecfa696c94b4f30 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-09T21:23:02,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/09bf9e6889e849089eb65717dd2561e3 is 1079, key is tmprow/info:/1731187382286/Put/seqid=0
2024-11-09T21:23:02,295 WARN  [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:02,295 WARN  [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK], DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]) is bad.
2024-11-09T21:23:02,295 WARN  [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741856_1039
2024-11-09T21:23:02,296 WARN  [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]
2024-11-09T21:23:02,297 WARN  [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:02,297 WARN  [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK], DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]) is bad.
2024-11-09T21:23:02,298 WARN  [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741857_1040
2024-11-09T21:23:02,298 WARN  [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]
2024-11-09T21:23:02,300 WARN  [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:02,300 WARN  [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK], DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]) is bad.
2024-11-09T21:23:02,300 WARN  [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741858_1041
2024-11-09T21:23:02,300 WARN  [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]
2024-11-09T21:23:02,303 WARN  [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:32917
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:02,303 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54548 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741859_1042] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data8]'}, localName='127.0.0.1:34361', datanodeUuid='141d9b99-a758-4387-9668-071f4ea5ccf7', xmitsInProgress=0}:Exception transferring block BP-1331531764-172.17.0.2-1731187359371:blk_1073741859_1042 to mirror 127.0.0.1:32917
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:02,303 WARN  [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK], DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]) is bad.
2024-11-09T21:23:02,303 WARN  [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741859_1042
2024-11-09T21:23:02,303 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54548 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741859_1042] {}] datanode.BlockReceiver(316): Block 1073741859 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-09T21:23:02,303 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54548 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741859_1042] {}] datanode.DataXceiver(331): 127.0.0.1:34361:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54548 dst: /127.0.0.1:34361
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:02,304 WARN  [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]
2024-11-09T21:23:02,305 WARN  [IPC Server handler 1 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-09T21:23:02,305 WARN  [IPC Server handler 1 on default port 42197 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-09T21:23:02,305 WARN  [IPC Server handler 1 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-09T21:23:02,309 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741860_1043 (size=6027)
2024-11-09T21:23:02,585 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880 is not closed yet, will try archiving it next time
2024-11-09T21:23:02,603 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:02,711 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/09bf9e6889e849089eb65717dd2561e3
2024-11-09T21:23:02,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/09bf9e6889e849089eb65717dd2561e3 as hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/09bf9e6889e849089eb65717dd2561e3
2024-11-09T21:23:02,731 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/09bf9e6889e849089eb65717dd2561e3, entries=1, sequenceid=34, filesize=5.9 K
2024-11-09T21:23:02,732 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 922e61fda0bc2cc7cecfa696c94b4f30 in 445ms, sequenceid=34, compaction requested=true
2024-11-09T21:23:02,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 922e61fda0bc2cc7cecfa696c94b4f30:

2024-11-09T21:23:02,732 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-11-09T21:23:02,732 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:23:02,732 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/78c276bdc48b41aa8b5f2ac16ce712e9 because midkey is the same as first or last row
2024-11-09T21:23:02,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 922e61fda0bc2cc7cecfa696c94b4f30:info, priority=-2147483648, current under compaction store size is 1
2024-11-09T21:23:02,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:23:02,733 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-09T21:23:02,734 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-09T21:23:02,734 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.HStore(1541): 922e61fda0bc2cc7cecfa696c94b4f30/info is initiating minor compaction (all files)
2024-11-09T21:23:02,734 INFO  [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 922e61fda0bc2cc7cecfa696c94b4f30/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.
2024-11-09T21:23:02,734 INFO  [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/748681af973648a89d55225e157d5749, hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/78c276bdc48b41aa8b5f2ac16ce712e9, hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/09bf9e6889e849089eb65717dd2561e3] into tmpdir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp, totalSize=28.2 K
2024-11-09T21:23:02,735 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 748681af973648a89d55225e157d5749, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731187376177
2024-11-09T21:23:02,735 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78c276bdc48b41aa8b5f2ac16ce712e9, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1731187380212
2024-11-09T21:23:02,736 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09bf9e6889e849089eb65717dd2561e3, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731187382286
2024-11-09T21:23:02,749 INFO  [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 922e61fda0bc2cc7cecfa696c94b4f30#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-09T21:23:02,750 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/39414d12a7894bba98e9405171e0ef34 is 1080, key is row0002/info:/1731187376177/Put/seqid=0
2024-11-09T21:23:02,752 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:02,752 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK], DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]) is bad.
2024-11-09T21:23:02,752 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741861_1044
2024-11-09T21:23:02,753 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]
2024-11-09T21:23:02,755 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:32917
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:02,755 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54584 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741862_1045] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data8]'}, localName='127.0.0.1:34361', datanodeUuid='141d9b99-a758-4387-9668-071f4ea5ccf7', xmitsInProgress=0}:Exception transferring block BP-1331531764-172.17.0.2-1731187359371:blk_1073741862_1045 to mirror 127.0.0.1:32917
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:02,755 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK], DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]) is bad.
2024-11-09T21:23:02,755 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54584 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741862_1045] {}] datanode.BlockReceiver(316): Block 1073741862 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-09T21:23:02,755 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741862_1045
2024-11-09T21:23:02,755 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54584 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741862_1045] {}] datanode.DataXceiver(331): 127.0.0.1:34361:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54584 dst: /127.0.0.1:34361
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:02,756 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]
2024-11-09T21:23:02,757 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:02,757 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK], DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]) is bad.
2024-11-09T21:23:02,757 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741863_1046
2024-11-09T21:23:02,758 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]
2024-11-09T21:23:02,760 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40849
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:02,760 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54598 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741864_1047] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data8]'}, localName='127.0.0.1:34361', datanodeUuid='141d9b99-a758-4387-9668-071f4ea5ccf7', xmitsInProgress=0}:Exception transferring block BP-1331531764-172.17.0.2-1731187359371:blk_1073741864_1047 to mirror 127.0.0.1:40849
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:02,760 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK], DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]) is bad.
2024-11-09T21:23:02,760 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54598 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741864_1047] {}] datanode.BlockReceiver(316): Block 1073741864 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-09T21:23:02,760 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741864_1047
2024-11-09T21:23:02,760 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54598 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741864_1047] {}] datanode.DataXceiver(331): 127.0.0.1:34361:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54598 dst: /127.0.0.1:34361
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:02,761 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]
2024-11-09T21:23:02,762 WARN  [IPC Server handler 0 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-09T21:23:02,762 WARN  [IPC Server handler 0 on default port 42197 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-09T21:23:02,762 WARN  [IPC Server handler 0 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-09T21:23:02,765 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741865_1048 (size=17994)
2024-11-09T21:23:03,176 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/39414d12a7894bba98e9405171e0ef34 as hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/39414d12a7894bba98e9405171e0ef34
2024-11-09T21:23:03,183 INFO  [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 922e61fda0bc2cc7cecfa696c94b4f30/info of 922e61fda0bc2cc7cecfa696c94b4f30 into 39414d12a7894bba98e9405171e0ef34(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-09T21:23:03,183 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 922e61fda0bc2cc7cecfa696c94b4f30:

2024-11-09T21:23:03,183 INFO  [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30., storeName=922e61fda0bc2cc7cecfa696c94b4f30/info, priority=13, startTime=1731187382732; duration=0sec
2024-11-09T21:23:03,183 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-09T21:23:03,183 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:23:03,183 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/39414d12a7894bba98e9405171e0ef34 because midkey is the same as first or last row
2024-11-09T21:23:03,184 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-09T21:23:03,184 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:23:03,184 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/39414d12a7894bba98e9405171e0ef34 because midkey is the same as first or last row
2024-11-09T21:23:03,184 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-09T21:23:03,184 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:23:03,184 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/39414d12a7894bba98e9405171e0ef34 because midkey is the same as first or last row
2024-11-09T21:23:03,184 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:23:03,184 DEBUG [RS:0;d86ff69d6719:37671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 922e61fda0bc2cc7cecfa696c94b4f30:info
2024-11-09T21:23:03,471 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37671 {}] regionserver.HRegion(8855): Flush requested on 922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:23:03,717 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 922e61fda0bc2cc7cecfa696c94b4f30 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-09T21:23:03,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/75f219399e784046b3a73b056ee18bb2 is 1079, key is tmprow/info:/1731187383714/Put/seqid=0
2024-11-09T21:23:03,724 WARN  [Thread-953 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:03,725 WARN  [Thread-953 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK], DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]) is bad.
2024-11-09T21:23:03,725 WARN  [Thread-953 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741866_1049
2024-11-09T21:23:03,725 WARN  [Thread-953 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]
2024-11-09T21:23:03,727 WARN  [Thread-953 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40849
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:03,727 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54622 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741867_1050] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data8]'}, localName='127.0.0.1:34361', datanodeUuid='141d9b99-a758-4387-9668-071f4ea5ccf7', xmitsInProgress=0}:Exception transferring block BP-1331531764-172.17.0.2-1731187359371:blk_1073741867_1050 to mirror 127.0.0.1:40849
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:03,728 WARN  [Thread-953 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK], DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]) is bad.
2024-11-09T21:23:03,728 WARN  [Thread-953 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741867_1050
2024-11-09T21:23:03,728 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54622 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741867_1050] {}] datanode.BlockReceiver(316): Block 1073741867 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-09T21:23:03,728 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1029235156_22 at /127.0.0.1:54622 [Receiving block BP-1331531764-172.17.0.2-1731187359371:blk_1073741867_1050] {}] datanode.DataXceiver(331): 127.0.0.1:34361:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54622 dst: /127.0.0.1:34361
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:03,728 WARN  [Thread-953 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]
2024-11-09T21:23:03,730 WARN  [Thread-953 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:03,730 WARN  [Thread-953 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK], DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]) is bad.
2024-11-09T21:23:03,730 WARN  [Thread-953 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741868_1051
2024-11-09T21:23:03,731 WARN  [Thread-953 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]
2024-11-09T21:23:03,732 WARN  [Thread-953 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:03,732 WARN  [Thread-953 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK], DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]) is bad.
2024-11-09T21:23:03,732 WARN  [Thread-953 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741869_1052
2024-11-09T21:23:03,732 WARN  [Thread-953 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]
2024-11-09T21:23:03,733 WARN  [IPC Server handler 3 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-09T21:23:03,733 WARN  [IPC Server handler 3 on default port 42197 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-09T21:23:03,733 WARN  [IPC Server handler 3 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-09T21:23:03,736 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741870_1053 (size=6027)
2024-11-09T21:23:04,137 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/75f219399e784046b3a73b056ee18bb2
2024-11-09T21:23:04,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/75f219399e784046b3a73b056ee18bb2 as hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/75f219399e784046b3a73b056ee18bb2
2024-11-09T21:23:04,155 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/75f219399e784046b3a73b056ee18bb2, entries=1, sequenceid=45, filesize=5.9 K
2024-11-09T21:23:04,156 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 922e61fda0bc2cc7cecfa696c94b4f30 in 440ms, sequenceid=45, compaction requested=false
2024-11-09T21:23:04,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 922e61fda0bc2cc7cecfa696c94b4f30:

2024-11-09T21:23:04,156 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-11-09T21:23:04,156 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:23:04,156 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/39414d12a7894bba98e9405171e0ef34 because midkey is the same as first or last row
2024-11-09T21:23:04,183 WARN  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK]]
2024-11-09T21:23:04,183 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:04,183 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d86ff69d6719%2C37671%2C1731187361371:(num 1731187382159) roll requested
2024-11-09T21:23:04,184 INFO  [regionserver/d86ff69d6719:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C37671%2C1731187361371.1731187384184
2024-11-09T21:23:04,187 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:04,187 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK], DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]) is bad.
2024-11-09T21:23:04,187 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741871_1054
2024-11-09T21:23:04,187 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]
2024-11-09T21:23:04,189 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:04,189 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK], DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]) is bad.
2024-11-09T21:23:04,189 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741872_1055
2024-11-09T21:23:04,189 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]
2024-11-09T21:23:04,190 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:04,190 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK], DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]) is bad.
2024-11-09T21:23:04,191 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741873_1056
2024-11-09T21:23:04,191 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]
2024-11-09T21:23:04,192 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:04,192 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK], DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]) is bad.
2024-11-09T21:23:04,192 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741874_1057
2024-11-09T21:23:04,193 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]
2024-11-09T21:23:04,194 WARN  [IPC Server handler 3 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-09T21:23:04,194 WARN  [IPC Server handler 3 on default port 42197 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-09T21:23:04,194 WARN  [IPC Server handler 3 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-09T21:23:04,196 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:04,196 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:04,197 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:04,197 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:04,197 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:04,197 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187382159 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187384184
2024-11-09T21:23:04,198 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37607:37607)]
2024-11-09T21:23:04,198 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880 is not closed yet, will try archiving it next time
2024-11-09T21:23:04,198 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187382159 is not closed yet, will try archiving it next time
2024-11-09T21:23:04,198 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187378136 to hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/oldWALs/d86ff69d6719%2C37671%2C1731187361371.1731187378136
2024-11-09T21:23:04,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741855_1038 (size=13591)
2024-11-09T21:23:04,601 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880 is not closed yet, will try archiving it next time
2024-11-09T21:23:04,603 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:04,931 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@14cfbac1[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:34361, datanodeUuid=141d9b99-a758-4387-9668-071f4ea5ccf7, infoPort=37607, infoSecurePort=0, ipcPort=39687, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371):Failed to transfer BP-1331531764-172.17.0.2-1731187359371:blk_1073741860_1043 to 127.0.0.1:44811 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:04,931 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@65249791[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:34361, datanodeUuid=141d9b99-a758-4387-9668-071f4ea5ccf7, infoPort=37607, infoSecurePort=0, ipcPort=39687, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371):Failed to transfer BP-1331531764-172.17.0.2-1731187359371:blk_1073741840_1023 to 127.0.0.1:32917 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37671 {}] regionserver.HRegion(8855): Flush requested on 922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:23:05,154 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 922e61fda0bc2cc7cecfa696c94b4f30 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-09T21:23:05,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/c4772831c63d4d1a922aabe8e95cbc0d is 1079, key is tmprow/info:/1731187385150/Put/seqid=0
2024-11-09T21:23:05,163 WARN  [Thread-962 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:05,163 WARN  [Thread-962 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK], DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]) is bad.
2024-11-09T21:23:05,163 WARN  [Thread-962 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741876_1059
2024-11-09T21:23:05,164 WARN  [Thread-962 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]
2024-11-09T21:23:05,165 WARN  [Thread-962 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:05,165 WARN  [Thread-962 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK], DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]) is bad.
2024-11-09T21:23:05,165 WARN  [Thread-962 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741877_1060
2024-11-09T21:23:05,165 WARN  [Thread-962 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]
2024-11-09T21:23:05,166 WARN  [Thread-962 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:05,167 WARN  [Thread-962 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK], DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]) is bad.
2024-11-09T21:23:05,167 WARN  [Thread-962 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741878_1061
2024-11-09T21:23:05,167 WARN  [Thread-962 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]
2024-11-09T21:23:05,168 WARN  [Thread-962 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:05,168 WARN  [Thread-962 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK], DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]) is bad.
2024-11-09T21:23:05,168 WARN  [Thread-962 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741879_1062
2024-11-09T21:23:05,169 WARN  [Thread-962 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]
2024-11-09T21:23:05,169 WARN  [IPC Server handler 3 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-09T21:23:05,169 WARN  [IPC Server handler 3 on default port 42197 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-09T21:23:05,170 WARN  [IPC Server handler 3 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-09T21:23:05,173 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741880_1063 (size=6027)
2024-11-09T21:23:05,472 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:05,575 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/c4772831c63d4d1a922aabe8e95cbc0d
2024-11-09T21:23:05,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/c4772831c63d4d1a922aabe8e95cbc0d as hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/c4772831c63d4d1a922aabe8e95cbc0d
2024-11-09T21:23:05,590 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/c4772831c63d4d1a922aabe8e95cbc0d, entries=1, sequenceid=55, filesize=5.9 K
2024-11-09T21:23:05,592 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 922e61fda0bc2cc7cecfa696c94b4f30 in 437ms, sequenceid=55, compaction requested=true
2024-11-09T21:23:05,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 922e61fda0bc2cc7cecfa696c94b4f30:

2024-11-09T21:23:05,592 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-11-09T21:23:05,592 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:23:05,592 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/39414d12a7894bba98e9405171e0ef34 because midkey is the same as first or last row
2024-11-09T21:23:05,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 922e61fda0bc2cc7cecfa696c94b4f30:info, priority=-2147483648, current under compaction store size is 1
2024-11-09T21:23:05,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:23:05,592 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-09T21:23:05,594 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-09T21:23:05,594 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.HStore(1541): 922e61fda0bc2cc7cecfa696c94b4f30/info is initiating minor compaction (all files)
2024-11-09T21:23:05,594 INFO  [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 922e61fda0bc2cc7cecfa696c94b4f30/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.
2024-11-09T21:23:05,594 INFO  [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/39414d12a7894bba98e9405171e0ef34, hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/75f219399e784046b3a73b056ee18bb2, hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/c4772831c63d4d1a922aabe8e95cbc0d] into tmpdir=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp, totalSize=29.3 K
2024-11-09T21:23:05,595 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] compactions.Compactor(225): Compacting 39414d12a7894bba98e9405171e0ef34, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731187376177
2024-11-09T21:23:05,595 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] compactions.Compactor(225): Compacting 75f219399e784046b3a73b056ee18bb2, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731187383714
2024-11-09T21:23:05,596 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] compactions.Compactor(225): Compacting c4772831c63d4d1a922aabe8e95cbc0d, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731187385150
2024-11-09T21:23:05,611 INFO  [RS:0;d86ff69d6719:37671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 922e61fda0bc2cc7cecfa696c94b4f30#info#compaction#24 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-09T21:23:05,611 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/6ecead493508413e915c197cb865733b is 1080, key is row0002/info:/1731187376177/Put/seqid=0
2024-11-09T21:23:05,613 WARN  [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:05,613 WARN  [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK], DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]) is bad.
2024-11-09T21:23:05,613 WARN  [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741881_1064
2024-11-09T21:23:05,614 WARN  [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]
2024-11-09T21:23:05,615 WARN  [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:05,615 WARN  [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK], DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]) is bad.
2024-11-09T21:23:05,615 WARN  [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741882_1065
2024-11-09T21:23:05,616 WARN  [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44811,DS-246beb66-219b-4473-b90c-a937cc6aa11e,DISK]
2024-11-09T21:23:05,617 WARN  [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:05,617 WARN  [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK], DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]) is bad.
2024-11-09T21:23:05,617 WARN  [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741883_1066
2024-11-09T21:23:05,618 WARN  [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]
2024-11-09T21:23:05,619 WARN  [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:05,619 WARN  [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK], DatanodeInfoWithStorage[127.0.0.1:34361,DS-68d83e8b-f808-47df-88af-73a926e3265c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]) is bad.
2024-11-09T21:23:05,619 WARN  [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741884_1067
2024-11-09T21:23:05,620 WARN  [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]
2024-11-09T21:23:05,620 WARN  [IPC Server handler 3 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-09T21:23:05,621 WARN  [IPC Server handler 3 on default port 42197 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-09T21:23:05,621 WARN  [IPC Server handler 3 on default port 42197 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-09T21:23:05,623 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741885_1068 (size=18097)
2024-11-09T21:23:05,932 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@65249791[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:34361, datanodeUuid=141d9b99-a758-4387-9668-071f4ea5ccf7, infoPort=37607, infoSecurePort=0, ipcPort=39687, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371):Failed to transfer BP-1331531764-172.17.0.2-1731187359371:blk_1073741865_1048 to 127.0.0.1:32917 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:05,932 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@14cfbac1[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:34361, datanodeUuid=141d9b99-a758-4387-9668-071f4ea5ccf7, infoPort=37607, infoSecurePort=0, ipcPort=39687, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371):Failed to transfer BP-1331531764-172.17.0.2-1731187359371:blk_1073741870_1053 to 127.0.0.1:44811 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:06,039 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/6ecead493508413e915c197cb865733b as hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/6ecead493508413e915c197cb865733b
2024-11-09T21:23:06,048 INFO  [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 922e61fda0bc2cc7cecfa696c94b4f30/info of 922e61fda0bc2cc7cecfa696c94b4f30 into 6ecead493508413e915c197cb865733b(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-09T21:23:06,048 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 922e61fda0bc2cc7cecfa696c94b4f30:

2024-11-09T21:23:06,048 INFO  [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30., storeName=922e61fda0bc2cc7cecfa696c94b4f30/info, priority=13, startTime=1731187385592; duration=0sec
2024-11-09T21:23:06,048 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-09T21:23:06,048 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:23:06,048 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/6ecead493508413e915c197cb865733b because midkey is the same as first or last row
2024-11-09T21:23:06,048 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-09T21:23:06,048 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:23:06,048 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/6ecead493508413e915c197cb865733b because midkey is the same as first or last row
2024-11-09T21:23:06,048 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-09T21:23:06,048 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:23:06,049 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/6ecead493508413e915c197cb865733b because midkey is the same as first or last row
2024-11-09T21:23:06,049 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:23:06,049 DEBUG [RS:0;d86ff69d6719:37671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 922e61fda0bc2cc7cecfa696c94b4f30:info
2024-11-09T21:23:06,199 WARN  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-11-09T21:23:06,199 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:06,401 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:23:06,404 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:23:06,405 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:23:06,405 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:23:06,405 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T21:23:06,406 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6e6ddf83{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:23:06,406 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@213456fa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:23:06,499 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6dbadaea{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/java.io.tmpdir/jetty-localhost-41513-hadoop-hdfs-3_4_1-tests_jar-_-any-10517070601607870530/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:23:06,500 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e48919f{HTTP/1.1, (http/1.1)}{localhost:41513}
2024-11-09T21:23:06,500 INFO  [Time-limited test {}] server.Server(415): Started @133919ms
2024-11-09T21:23:06,501 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:23:06,604 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:06,841 WARN  [Thread-984 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:23:06,844 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd04670fea5cb3b05 with lease ID 0xfd543dc177bc545f: from storage DS-246beb66-219b-4473-b90c-a937cc6aa11e node DatanodeRegistration(127.0.0.1:37129, datanodeUuid=2894e6a3-288e-408f-8bd5-b84a909e5593, infoPort=42465, infoSecurePort=0, ipcPort=37431, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-09T21:23:06,845 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd04670fea5cb3b05 with lease ID 0xfd543dc177bc545f: from storage DS-5a4e1575-1fa4-460a-ac37-4380e7ac604e node DatanodeRegistration(127.0.0.1:37129, datanodeUuid=2894e6a3-288e-408f-8bd5-b84a909e5593, infoPort=42465, infoSecurePort=0, ipcPort=37431, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:23:07,472 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:07,933 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@65249791[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:34361, datanodeUuid=141d9b99-a758-4387-9668-071f4ea5ccf7, infoPort=37607, infoSecurePort=0, ipcPort=39687, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371):Failed to transfer BP-1331531764-172.17.0.2-1731187359371:blk_1073741880_1063 to 127.0.0.1:36141 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:07,933 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@14cfbac1[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:34361, datanodeUuid=141d9b99-a758-4387-9668-071f4ea5ccf7, infoPort=37607, infoSecurePort=0, ipcPort=39687, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371):Failed to transfer BP-1331531764-172.17.0.2-1731187359371:blk_1073741855_1038 to 127.0.0.1:36141 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:08,199 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:08,605 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:08,936 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37129 is added to blk_1073741885_1068 (size=18097)
2024-11-09T21:23:09,473 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:10,200 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:10,605 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:11,210 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T21:23:11,473 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:11,676 ERROR [FSHLog-0-hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData-prefix:d86ff69d6719,33537,1731187361230 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:11,676 WARN  [FSHLog-0-hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData-prefix:d86ff69d6719,33537,1731187361230 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:11,677 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog d86ff69d6719%2C33537%2C1731187361230:(num 1731187361488) roll requested
2024-11-09T21:23:11,678 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C33537%2C1731187361230.1731187391677
2024-11-09T21:23:11,684 WARN  [Thread-1003 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1069
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:11,684 WARN  [Thread-1003 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741886_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK], DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]) is bad.
2024-11-09T21:23:11,684 WARN  [Thread-1003 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741886_1069
2024-11-09T21:23:11,685 WARN  [Thread-1003 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]
2024-11-09T21:23:11,687 WARN  [Thread-1003 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1070
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:11,687 WARN  [Thread-1003 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1331531764-172.17.0.2-1731187359371:blk_1073741887_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK], DatanodeInfoWithStorage[127.0.0.1:36141,DS-026c0227-9d92-4566-9f41-fc21b2c4222d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]) is bad.
2024-11-09T21:23:11,687 WARN  [Thread-1003 {}] hdfs.DataStreamer(1850): Abandoning BP-1331531764-172.17.0.2-1731187359371:blk_1073741887_1070
2024-11-09T21:23:11,688 WARN  [Thread-1003 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40849,DS-f772052f-7fcd-4837-b069-59cfbccaba2b,DISK]
2024-11-09T21:23:11,692 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:11,693 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:11,693 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:11,693 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:11,693 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:11,693 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/WALs/d86ff69d6719,33537,1731187361230/d86ff69d6719%2C33537%2C1731187361230.1731187361488 with entries=54, filesize=26.68 KB; new WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/WALs/d86ff69d6719,33537,1731187361230/d86ff69d6719%2C33537%2C1731187361230.1731187391677
2024-11-09T21:23:11,694 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:11,694 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:11,694 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/WALs/d86ff69d6719,33537,1731187361230/d86ff69d6719%2C33537%2C1731187361230.1731187361488
2024-11-09T21:23:11,695 WARN  [IPC Server handler 3 on default port 42197 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/WALs/d86ff69d6719,33537,1731187361230/d86ff69d6719%2C33537%2C1731187361230.1731187361488 has not been closed. Lease recovery is in progress. RecoveryId = 1072 for block blk_1073741830_1006
2024-11-09T21:23:11,695 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/WALs/d86ff69d6719,33537,1731187361230/d86ff69d6719%2C33537%2C1731187361230.1731187361488 after 1ms
2024-11-09T21:23:11,696 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42465:42465),(127.0.0.1/127.0.0.1:37607:37607)]
2024-11-09T21:23:11,696 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/WALs/d86ff69d6719,33537,1731187361230/d86ff69d6719%2C33537%2C1731187361230.1731187361488 is not closed yet, will try archiving it next time
2024-11-09T21:23:12,201 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:12,606 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:14,201 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:14,607 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:15,698 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/WALs/d86ff69d6719,33537,1731187361230/d86ff69d6719%2C33537%2C1731187361230.1731187361488 after 4004ms
2024-11-09T21:23:16,202 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:16,607 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:16,849 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@44467e67[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37129, datanodeUuid=2894e6a3-288e-408f-8bd5-b84a909e5593, infoPort=42465, infoSecurePort=0, ipcPort=37431, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371):Failed to transfer BP-1331531764-172.17.0.2-1731187359371:blk_1073741836_1012 to 127.0.0.1:36141 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:16,854 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741832_1008 (size=32)
2024-11-09T21:23:17,849 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@13f34436[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37129, datanodeUuid=2894e6a3-288e-408f-8bd5-b84a909e5593, infoPort=42465, infoSecurePort=0, ipcPort=37431, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371):Failed to transfer BP-1331531764-172.17.0.2-1731187359371:blk_1073741828_1004 to 127.0.0.1:36141 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:17,849 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@44467e67[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37129, datanodeUuid=2894e6a3-288e-408f-8bd5-b84a909e5593, infoPort=42465, infoSecurePort=0, ipcPort=37431, storageInfo=lv=-57;cid=testClusterID;nsid=674563138;c=1731187359371):Failed to transfer BP-1331531764-172.17.0.2-1731187359371:blk_1073741826_1002 to 127.0.0.1:40849 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:18,203 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:18,608 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:19,855 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741825_1001 (size=7)
2024-11-09T21:23:20,124 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C37671%2C1731187361371.1731187400124
2024-11-09T21:23:20,133 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,133 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,133 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,133 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,134 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,134 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187384184 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187400124
2024-11-09T21:23:20,135 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42465:42465),(127.0.0.1/127.0.0.1:37607:37607)]
2024-11-09T21:23:20,135 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880 is not closed yet, will try archiving it next time
2024-11-09T21:23:20,135 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187384184 is not closed yet, will try archiving it next time
2024-11-09T21:23:20,136 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187382159 to hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/oldWALs/d86ff69d6719%2C37671%2C1731187361371.1731187382159
2024-11-09T21:23:20,136 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741875_1058 (size=12911)
2024-11-09T21:23:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37671 {}] regionserver.HRegion(8855): Flush requested on 922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:23:20,141 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 922e61fda0bc2cc7cecfa696c94b4f30 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-09T21:23:20,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/cf601c69d06b4c10b19f3a98e7e0951f is 1080, key is row0013/info:/1731187400137/Put/seqid=0
2024-11-09T21:23:20,157 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37129 is added to blk_1073741890_1074 (size=8190)
2024-11-09T21:23:20,157 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741890_1074 (size=8190)
2024-11-09T21:23:20,158 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/cf601c69d06b4c10b19f3a98e7e0951f
2024-11-09T21:23:20,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/cf601c69d06b4c10b19f3a98e7e0951f as hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/cf601c69d06b4c10b19f3a98e7e0951f
2024-11-09T21:23:20,172 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/cf601c69d06b4c10b19f3a98e7e0951f, entries=3, sequenceid=66, filesize=8.0 K
2024-11-09T21:23:20,173 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for 922e61fda0bc2cc7cecfa696c94b4f30 in 32ms, sequenceid=66, compaction requested=false
2024-11-09T21:23:20,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 922e61fda0bc2cc7cecfa696c94b4f30:

2024-11-09T21:23:20,174 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K
2024-11-09T21:23:20,174 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:23:20,174 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/6ecead493508413e915c197cb865733b because midkey is the same as first or last row
2024-11-09T21:23:20,204 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-11-09T21:23:20,204 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:20,363 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-09T21:23:20,363 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-09T21:23:20,364 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:23:20,364 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:23:20,364 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:23:20,364 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-09T21:23:20,364 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-09T21:23:20,364 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1890729843, stopped=false
2024-11-09T21:23:20,365 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d86ff69d6719,33537,1731187361230
2024-11-09T21:23:20,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-09T21:23:20,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-09T21:23:20,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:20,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:20,424 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-09T21:23:20,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x10122c458830002, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-09T21:23:20,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x10122c458830002, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:20,425 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-09T21:23:20,426 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:23:20,426 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:23:20,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:23:20,426 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:23:20,427 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40715-0x10122c458830002, quorum=127.0.0.1:54077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:23:20,427 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd86ff69d6719,37671,1731187361371' *****
2024-11-09T21:23:20,428 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-09T21:23:20,428 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd86ff69d6719,40715,1731187362517' *****
2024-11-09T21:23:20,428 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-09T21:23:20,428 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-09T21:23:20,428 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-09T21:23:20,429 INFO  [RS:0;d86ff69d6719:37671 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-09T21:23:20,429 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-09T21:23:20,429 INFO  [RS:0;d86ff69d6719:37671 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-09T21:23:20,429 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-09T21:23:20,429 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer(3091): Received CLOSE for 922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:23:20,430 INFO  [RS:1;d86ff69d6719:40715 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-09T21:23:20,430 INFO  [RS:1;d86ff69d6719:40715 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-09T21:23:20,430 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.HRegionServer(959): stopping server d86ff69d6719,40715,1731187362517
2024-11-09T21:23:20,430 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-09T21:23:20,430 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer(959): stopping server d86ff69d6719,37671,1731187361371
2024-11-09T21:23:20,430 INFO  [RS:1;d86ff69d6719:40715 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;d86ff69d6719:40715.
2024-11-09T21:23:20,430 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-09T21:23:20,431 INFO  [RS:0;d86ff69d6719:37671 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d86ff69d6719:37671.
2024-11-09T21:23:20,431 DEBUG [RS:1;d86ff69d6719:40715 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:23:20,431 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 922e61fda0bc2cc7cecfa696c94b4f30, disabling compactions & flushes
2024-11-09T21:23:20,431 DEBUG [RS:1;d86ff69d6719:40715 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:23:20,431 DEBUG [RS:0;d86ff69d6719:37671 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:23:20,431 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.
2024-11-09T21:23:20,431 DEBUG [RS:0;d86ff69d6719:37671 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:23:20,431 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.HRegionServer(976): stopping server d86ff69d6719,40715,1731187362517; all regions closed.
2024-11-09T21:23:20,431 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.
2024-11-09T21:23:20,432 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-09T21:23:20,432 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30. after waiting 0 ms
2024-11-09T21:23:20,432 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-09T21:23:20,432 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.
2024-11-09T21:23:20,432 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-09T21:23:20,432 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-09T21:23:20,432 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 922e61fda0bc2cc7cecfa696c94b4f30 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-09T21:23:20,432 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-09T21:23:20,432 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,432 DEBUG [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer(1325): Online Regions={922e61fda0bc2cc7cecfa696c94b4f30=TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30., 1588230740=hbase:meta,,1.1588230740}
2024-11-09T21:23:20,432 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,432 DEBUG [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 922e61fda0bc2cc7cecfa696c94b4f30
2024-11-09T21:23:20,432 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,433 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,433 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-09T21:23:20,433 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,433 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-09T21:23:20,433 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-09T21:23:20,433 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-09T21:23:20,433 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-09T21:23:20,433 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-11-09T21:23:20,433 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:20,433 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:20,433 ERROR [FSHLog-0-hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a-prefix:d86ff69d6719,37671,1731187361371.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:20,433 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
2024-11-09T21:23:20,433 WARN  [FSHLog-0-hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a-prefix:d86ff69d6719,37671,1731187361371.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:20,434 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d86ff69d6719%2C37671%2C1731187361371.meta:.meta(num 1731187362371) roll requested
2024-11-09T21:23:20,434 INFO  [regionserver/d86ff69d6719:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C37671%2C1731187361371.meta.1731187400434.meta
2024-11-09T21:23:20,434 WARN  [IPC Server handler 4 on default port 42197 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735 has not been closed. Lease recovery is in progress. RecoveryId = 1075 for block blk_1073741837_1013
2024-11-09T21:23:20,434 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735 after 1ms
2024-11-09T21:23:20,438 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/8aa60f7eddef4d228543027178bb07ed is 1080, key is row0015/info:/1731187400142/Put/seqid=0
2024-11-09T21:23:20,439 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,439 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,439 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,439 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,439 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,439 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187400434.meta
2024-11-09T21:23:20,440 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:20,440 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32917,DS-b95a08ae-1273-4a9d-a571-3bae263f6bb6,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:20,440 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
2024-11-09T21:23:20,441 WARN  [IPC Server handler 3 on default port 42197 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta has not been closed. Lease recovery is in progress. RecoveryId = 1078 for block blk_1073741834_1010
2024-11-09T21:23:20,441 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta after 1ms
2024-11-09T21:23:20,446 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37607:37607),(127.0.0.1/127.0.0.1:42465:42465)]
2024-11-09T21:23:20,446 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta is not closed yet, will try archiving it next time
2024-11-09T21:23:20,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741892_1077 (size=14660)
2024-11-09T21:23:20,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37129 is added to blk_1073741892_1077 (size=14660)
2024-11-09T21:23:20,448 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/8aa60f7eddef4d228543027178bb07ed
2024-11-09T21:23:20,454 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/.tmp/info/8aa60f7eddef4d228543027178bb07ed as hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/8aa60f7eddef4d228543027178bb07ed
2024-11-09T21:23:20,460 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/8aa60f7eddef4d228543027178bb07ed, entries=9, sequenceid=78, filesize=14.3 K
2024-11-09T21:23:20,462 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9683, heapSize ~10.36 KB/10608, currentSize=0 B/0 for 922e61fda0bc2cc7cecfa696c94b4f30 in 30ms, sequenceid=78, compaction requested=true
2024-11-09T21:23:20,463 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/748681af973648a89d55225e157d5749, hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/78c276bdc48b41aa8b5f2ac16ce712e9, hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/39414d12a7894bba98e9405171e0ef34, hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/09bf9e6889e849089eb65717dd2561e3, hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/75f219399e784046b3a73b056ee18bb2, hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/c4772831c63d4d1a922aabe8e95cbc0d] to archive
2024-11-09T21:23:20,464 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-09T21:23:20,466 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/.tmp/info/9a63e5605b4b4ea2b17900fa431ed550 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30./info:regioninfo/1731187363002/Put/seqid=0
2024-11-09T21:23:20,466 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/748681af973648a89d55225e157d5749 to hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/748681af973648a89d55225e157d5749
2024-11-09T21:23:20,467 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/78c276bdc48b41aa8b5f2ac16ce712e9 to hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/78c276bdc48b41aa8b5f2ac16ce712e9
2024-11-09T21:23:20,469 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/39414d12a7894bba98e9405171e0ef34 to hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/39414d12a7894bba98e9405171e0ef34
2024-11-09T21:23:20,470 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/09bf9e6889e849089eb65717dd2561e3 to hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/09bf9e6889e849089eb65717dd2561e3
2024-11-09T21:23:20,472 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/75f219399e784046b3a73b056ee18bb2 to hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/75f219399e784046b3a73b056ee18bb2
2024-11-09T21:23:20,473 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/c4772831c63d4d1a922aabe8e95cbc0d to hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/info/c4772831c63d4d1a922aabe8e95cbc0d
2024-11-09T21:23:20,474 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d86ff69d6719:33537 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-09T21:23:20,474 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [748681af973648a89d55225e157d5749=10347, 78c276bdc48b41aa8b5f2ac16ce712e9=12506, 39414d12a7894bba98e9405171e0ef34=17994, 09bf9e6889e849089eb65717dd2561e3=6027, 75f219399e784046b3a73b056ee18bb2=6027, c4772831c63d4d1a922aabe8e95cbc0d=6027]
2024-11-09T21:23:20,480 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741893_1079 (size=7089)
2024-11-09T21:23:20,480 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37129 is added to blk_1073741893_1079 (size=7089)
2024-11-09T21:23:20,481 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/.tmp/info/9a63e5605b4b4ea2b17900fa431ed550
2024-11-09T21:23:20,482 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/922e61fda0bc2cc7cecfa696c94b4f30/recovered.edits/81.seqid, newMaxSeqId=81, maxSeqId=1
2024-11-09T21:23:20,483 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.
2024-11-09T21:23:20,483 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 922e61fda0bc2cc7cecfa696c94b4f30:
Waiting for close lock at 1731187400431Running coprocessor pre-close hooks at 1731187400431Disabling compacts and flushes for region at 1731187400431Disabling writes for close at 1731187400432 (+1 ms)Obtaining lock to block concurrent updates at 1731187400432Preparing flush snapshotting stores in 922e61fda0bc2cc7cecfa696c94b4f30 at 1731187400432Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30., syncing WAL and waiting on mvcc, flushsize=dataSize=9683, getHeapSize=10608, getOffHeapSize=0, getCellsCount=9 at 1731187400432Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30. at 1731187400433 (+1 ms)Flushing 922e61fda0bc2cc7cecfa696c94b4f30/info: creating writer at 1731187400433Flushing 922e61fda0bc2cc7cecfa696c94b4f30/info: appending metadata at 1731187400437 (+4 ms)Flushing 922e61fda0bc2cc7cecfa696c94b4f30/info: closing flushed file at 1731187400437Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@387def50: reopening flushed file at 1731187400453 (+16 ms)Finished flush of dataSize ~9.46 KB/9683, heapSize ~10.36 KB/10608, currentSize=0 B/0 for 922e61fda0bc2cc7cecfa696c94b4f30 in 30ms, sequenceid=78, compaction requested=true at 1731187400462 (+9 ms)Writing region close event to WAL at 1731187400479 (+17 ms)Running coprocessor post-close hooks at 1731187400483 (+4 ms)Closed at 1731187400483
2024-11-09T21:23:20,484 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731187362629.922e61fda0bc2cc7cecfa696c94b4f30.
2024-11-09T21:23:20,501 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/.tmp/ns/379d1a9829a346beb591417b9c2a03c4 is 43, key is default/ns:d/1731187362468/Put/seqid=0
2024-11-09T21:23:20,506 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741894_1080 (size=5153)
2024-11-09T21:23:20,506 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37129 is added to blk_1073741894_1080 (size=5153)
2024-11-09T21:23:20,507 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/.tmp/ns/379d1a9829a346beb591417b9c2a03c4
2024-11-09T21:23:20,526 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/.tmp/table/f11c7d12a3d34149a19af125b68eb3ec is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1731187363015/Put/seqid=0
2024-11-09T21:23:20,531 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37129 is added to blk_1073741895_1081 (size=5424)
2024-11-09T21:23:20,531 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741895_1081 (size=5424)
2024-11-09T21:23:20,532 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/.tmp/table/f11c7d12a3d34149a19af125b68eb3ec
2024-11-09T21:23:20,537 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880 is not closed yet, will try archiving it next time
2024-11-09T21:23:20,537 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187384184 to hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/oldWALs/d86ff69d6719%2C37671%2C1731187361371.1731187384184
2024-11-09T21:23:20,538 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/.tmp/info/9a63e5605b4b4ea2b17900fa431ed550 as hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/info/9a63e5605b4b4ea2b17900fa431ed550
2024-11-09T21:23:20,544 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/info/9a63e5605b4b4ea2b17900fa431ed550, entries=10, sequenceid=11, filesize=6.9 K
2024-11-09T21:23:20,545 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/.tmp/ns/379d1a9829a346beb591417b9c2a03c4 as hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/ns/379d1a9829a346beb591417b9c2a03c4
2024-11-09T21:23:20,552 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/ns/379d1a9829a346beb591417b9c2a03c4, entries=2, sequenceid=11, filesize=5.0 K
2024-11-09T21:23:20,553 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/.tmp/table/f11c7d12a3d34149a19af125b68eb3ec as hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/table/f11c7d12a3d34149a19af125b68eb3ec
2024-11-09T21:23:20,560 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/table/f11c7d12a3d34149a19af125b68eb3ec, entries=2, sequenceid=11, filesize=5.3 K
2024-11-09T21:23:20,561 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 128ms, sequenceid=11, compaction requested=false
2024-11-09T21:23:20,568 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-09T21:23:20,568 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-09T21:23:20,568 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-09T21:23:20,569 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731187400432Running coprocessor pre-close hooks at 1731187400432Disabling compacts and flushes for region at 1731187400433 (+1 ms)Disabling writes for close at 1731187400433Obtaining lock to block concurrent updates at 1731187400433Preparing flush snapshotting stores in 1588230740 at 1731187400433Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1731187400433Flushing stores of hbase:meta,,1.1588230740 at 1731187400446 (+13 ms)Flushing 1588230740/info: creating writer at 1731187400446Flushing 1588230740/info: appending metadata at 1731187400465 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731187400465Flushing 1588230740/ns: creating writer at 1731187400486 (+21 ms)Flushing 1588230740/ns: appending metadata at 1731187400500 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731187400500Flushing 1588230740/table: creating writer at 1731187400512 (+12 ms)Flushing 1588230740/table: appending metadata at 1731187400526 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731187400526Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35cf3b8b: reopening flushed file at 1731187400537 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@331b0b1d: reopening flushed file at 1731187400544 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41dfeecb: reopening flushed file at 1731187400552 (+8 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 128ms, sequenceid=11, compaction requested=false at 1731187400561 (+9 ms)Writing region close event to WAL at 1731187400564 (+3 ms)Running coprocessor post-close hooks at 1731187400568 (+4 ms)Closed at 1731187400568
2024-11-09T21:23:20,569 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-09T21:23:20,602 INFO  [regionserver/d86ff69d6719:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-09T21:23:20,633 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer(976): stopping server d86ff69d6719,37671,1731187361371; all regions closed.
2024-11-09T21:23:20,633 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,633 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,633 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,634 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,634 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:20,637 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37129 is added to blk_1073741891_1076 (size=825)
2024-11-09T21:23:20,638 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741891_1076 (size=825)
2024-11-09T21:23:20,651 INFO  [regionserver/d86ff69d6719:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-09T21:23:20,652 INFO  [regionserver/d86ff69d6719:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-09T21:23:20,742 INFO  [regionserver/d86ff69d6719:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-09T21:23:20,743 INFO  [regionserver/d86ff69d6719:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-09T21:23:21,749 INFO  [regionserver/d86ff69d6719:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-09T21:23:21,874 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@221b82a2 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1331531764-172.17.0.2-1731187359371:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:32917,null,null])
java.net.ConnectException: Call From d86ff69d6719/172.17.0.2 to localhost:46751 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-09T21:23:21,882 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37129 is added to blk_1073741830_1072 (size=27318)
2024-11-09T21:23:22,855 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741835_1011 (size=393)
2024-11-09T21:23:22,855 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741831_1007 (size=1321)
2024-11-09T21:23:23,010 INFO  [master/d86ff69d6719:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-09T21:23:23,010 INFO  [master/d86ff69d6719:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-09T21:23:23,855 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741829_1005 (size=34)
2024-11-09T21:23:23,855 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741827_1003 (size=196)
2024-11-09T21:23:24,435 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735 after 4002ms
2024-11-09T21:23:24,442 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta after 4002ms
2024-11-09T21:23:25,434 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-09T21:23:25,440 DEBUG [RS:1;d86ff69d6719:40715 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/oldWALs
2024-11-09T21:23:25,440 INFO  [RS:1;d86ff69d6719:40715 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d86ff69d6719%2C40715%2C1731187362517:(num 1731187362735)
2024-11-09T21:23:25,441 DEBUG [RS:1;d86ff69d6719:40715 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:23:25,441 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.LeaseManager(133): Closed leases
2024-11-09T21:23:25,441 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-09T21:23:25,442 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.ChoreService(370): Chore service for: regionserver/d86ff69d6719:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-09T21:23:25,442 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-09T21:23:25,442 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-09T21:23:25,443 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-09T21:23:25,443 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-09T21:23:25,443 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-09T21:23:25,443 INFO  [RS:1;d86ff69d6719:40715 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40715
2024-11-09T21:23:25,485 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:25,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x10122c458830002, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d86ff69d6719,40715,1731187362517
2024-11-09T21:23:25,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T21:23:25,489 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-09T21:23:25,491 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d86ff69d6719,40715,1731187362517]
2024-11-09T21:23:25,505 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:25,506 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d86ff69d6719,40715,1731187362517 already deleted, retry=false
2024-11-09T21:23:25,506 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:25,506 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d86ff69d6719,40715,1731187362517 expired; onlineServers=1
2024-11-09T21:23:25,506 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:25,506 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:25,506 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:25,514 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:25,514 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:25,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x10122c458830002, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:23:25,597 INFO  [RS:1;d86ff69d6719:40715 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-09T21:23:25,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40715-0x10122c458830002, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:23:25,598 INFO  [RS:1;d86ff69d6719:40715 {}] regionserver.HRegionServer(1031): Exiting; stopping=d86ff69d6719,40715,1731187362517; zookeeper connection closed.
2024-11-09T21:23:25,598 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3ca621d4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3ca621d4
2024-11-09T21:23:25,634 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-09T21:23:25,638 DEBUG [RS:0;d86ff69d6719:37671 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/oldWALs
2024-11-09T21:23:25,638 INFO  [RS:0;d86ff69d6719:37671 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d86ff69d6719%2C37671%2C1731187361371.meta:.meta(num 1731187400434)
2024-11-09T21:23:25,638 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:25,639 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:25,639 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:25,639 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:25,639 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:25,640 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741889_1073 (size=14682)
2024-11-09T21:23:25,641 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37129 is added to blk_1073741889_1073 (size=14682)
2024-11-09T21:23:25,851 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741830_1072 (size=27318)
2024-11-09T21:23:25,938 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37129 is added to blk_1073741875_1058 (size=12911)
2024-11-09T21:23:26,017 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-09T21:23:26,047 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:26,047 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:26,047 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:26,048 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:26,048 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:26,049 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:26,053 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:26,055 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:26,438 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:26,447 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-09T21:23:26,452 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:27,440 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:27,454 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:28,442 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:28,448 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-09T21:23:28,458 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:29,443 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:29,463 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:29,563 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-09T21:23:29,564 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-09T21:23:29,564 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-09T21:23:30,445 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:30,448 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-09T21:23:30,467 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:30,639 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-09T21:23:30,694 DEBUG [RS:0;d86ff69d6719:37671 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/oldWALs
2024-11-09T21:23:30,694 INFO  [RS:0;d86ff69d6719:37671 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d86ff69d6719%2C37671%2C1731187361371:(num 1731187400124)
2024-11-09T21:23:30,694 DEBUG [RS:0;d86ff69d6719:37671 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:23:30,694 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.LeaseManager(133): Closed leases
2024-11-09T21:23:30,695 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-09T21:23:30,695 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.ChoreService(370): Chore service for: regionserver/d86ff69d6719:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-09T21:23:30,696 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-09T21:23:30,696 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-09T21:23:30,697 INFO  [RS:0;d86ff69d6719:37671 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37671
2024-11-09T21:23:30,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d86ff69d6719,37671,1731187361371
2024-11-09T21:23:30,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T21:23:30,727 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-09T21:23:30,728 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d86ff69d6719,37671,1731187361371]
2024-11-09T21:23:30,747 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d86ff69d6719,37671,1731187361371 already deleted, retry=false
2024-11-09T21:23:30,747 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d86ff69d6719,37671,1731187361371 expired; onlineServers=0
2024-11-09T21:23:30,747 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd86ff69d6719,33537,1731187361230' *****
2024-11-09T21:23:30,747 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-09T21:23:30,748 INFO  [M:0;d86ff69d6719:33537 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-09T21:23:30,748 INFO  [M:0;d86ff69d6719:33537 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-09T21:23:30,748 DEBUG [M:0;d86ff69d6719:33537 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-09T21:23:30,749 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-09T21:23:30,749 DEBUG [M:0;d86ff69d6719:33537 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-09T21:23:30,749 DEBUG [master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187361675 {}] cleaner.HFileCleaner(306): Exit Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187361675,5,FailOnTimeoutGroup]
2024-11-09T21:23:30,749 DEBUG [master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187361675 {}] cleaner.HFileCleaner(306): Exit Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187361675,5,FailOnTimeoutGroup]
2024-11-09T21:23:30,749 INFO  [M:0;d86ff69d6719:33537 {}] hbase.ChoreService(370): Chore service for: master/d86ff69d6719:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-09T21:23:30,749 INFO  [M:0;d86ff69d6719:33537 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-09T21:23:30,750 DEBUG [M:0;d86ff69d6719:33537 {}] master.HMaster(1795): Stopping service threads
2024-11-09T21:23:30,750 INFO  [M:0;d86ff69d6719:33537 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-09T21:23:30,750 INFO  [M:0;d86ff69d6719:33537 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-09T21:23:30,750 INFO  [M:0;d86ff69d6719:33537 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-09T21:23:30,751 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-09T21:23:30,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-09T21:23:30,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:30,760 DEBUG [M:0;d86ff69d6719:33537 {}] zookeeper.ZKUtil(347): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-09T21:23:30,760 WARN  [M:0;d86ff69d6719:33537 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-09T21:23:30,761 INFO  [M:0;d86ff69d6719:33537 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/.lastflushedseqids
2024-11-09T21:23:30,767 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37129 is added to blk_1073741896_1082 (size=130)
2024-11-09T21:23:30,767 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741896_1082 (size=130)
2024-11-09T21:23:30,768 INFO  [M:0;d86ff69d6719:33537 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-09T21:23:30,768 INFO  [M:0;d86ff69d6719:33537 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-09T21:23:30,768 DEBUG [M:0;d86ff69d6719:33537 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-09T21:23:30,768 INFO  [M:0;d86ff69d6719:33537 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:23:30,768 DEBUG [M:0;d86ff69d6719:33537 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:23:30,768 DEBUG [M:0;d86ff69d6719:33537 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-09T21:23:30,768 DEBUG [M:0;d86ff69d6719:33537 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:23:30,769 INFO  [M:0;d86ff69d6719:33537 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.26 KB heapSize=29.50 KB
2024-11-09T21:23:30,786 DEBUG [M:0;d86ff69d6719:33537 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9d40ae1054194dc3b4265240293d71f0 is 82, key is hbase:meta,,1/info:regioninfo/1731187362406/Put/seqid=0
2024-11-09T21:23:30,791 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37129 is added to blk_1073741897_1083 (size=5672)
2024-11-09T21:23:30,791 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741897_1083 (size=5672)
2024-11-09T21:23:30,791 INFO  [M:0;d86ff69d6719:33537 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9d40ae1054194dc3b4265240293d71f0
2024-11-09T21:23:30,813 DEBUG [M:0;d86ff69d6719:33537 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1c3b13e5bfc94bbea1493dcbc91b648c is 775, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731187363022/Put/seqid=0
2024-11-09T21:23:30,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37129 is added to blk_1073741898_1084 (size=6256)
2024-11-09T21:23:30,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741898_1084 (size=6256)
2024-11-09T21:23:30,819 INFO  [M:0;d86ff69d6719:33537 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.59 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1c3b13e5bfc94bbea1493dcbc91b648c
2024-11-09T21:23:30,824 INFO  [M:0;d86ff69d6719:33537 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1c3b13e5bfc94bbea1493dcbc91b648c
2024-11-09T21:23:30,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:23:30,839 INFO  [RS:0;d86ff69d6719:37671 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-09T21:23:30,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37671-0x10122c458830001, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:23:30,839 INFO  [RS:0;d86ff69d6719:37671 {}] regionserver.HRegionServer(1031): Exiting; stopping=d86ff69d6719,37671,1731187361371; zookeeper connection closed.
2024-11-09T21:23:30,839 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@186acd3d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@186acd3d
2024-11-09T21:23:30,839 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-11-09T21:23:30,839 DEBUG [M:0;d86ff69d6719:33537 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/100c6ca9cfbf4410b5887921f2aded32 is 69, key is d86ff69d6719,37671,1731187361371/rs:state/1731187361725/Put/seqid=0
2024-11-09T21:23:30,844 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741899_1085 (size=5224)
2024-11-09T21:23:30,844 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37129 is added to blk_1073741899_1085 (size=5224)
2024-11-09T21:23:30,844 INFO  [M:0;d86ff69d6719:33537 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/100c6ca9cfbf4410b5887921f2aded32
2024-11-09T21:23:30,864 DEBUG [M:0;d86ff69d6719:33537 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/912fdc37e1224dd0860045ae6b51b278 is 52, key is load_balancer_on/state:d/1731187362503/Put/seqid=0
2024-11-09T21:23:30,869 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37129 is added to blk_1073741900_1086 (size=5056)
2024-11-09T21:23:30,869 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741900_1086 (size=5056)
2024-11-09T21:23:31,187 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:31,272 INFO  [M:0;d86ff69d6719:33537 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/912fdc37e1224dd0860045ae6b51b278
2024-11-09T21:23:31,285 DEBUG [M:0;d86ff69d6719:33537 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9d40ae1054194dc3b4265240293d71f0 as hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9d40ae1054194dc3b4265240293d71f0
2024-11-09T21:23:31,291 INFO  [M:0;d86ff69d6719:33537 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9d40ae1054194dc3b4265240293d71f0, entries=8, sequenceid=60, filesize=5.5 K
2024-11-09T21:23:31,292 DEBUG [M:0;d86ff69d6719:33537 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1c3b13e5bfc94bbea1493dcbc91b648c as hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1c3b13e5bfc94bbea1493dcbc91b648c
2024-11-09T21:23:31,298 INFO  [M:0;d86ff69d6719:33537 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1c3b13e5bfc94bbea1493dcbc91b648c
2024-11-09T21:23:31,298 INFO  [M:0;d86ff69d6719:33537 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1c3b13e5bfc94bbea1493dcbc91b648c, entries=6, sequenceid=60, filesize=6.1 K
2024-11-09T21:23:31,300 DEBUG [M:0;d86ff69d6719:33537 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/100c6ca9cfbf4410b5887921f2aded32 as hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/100c6ca9cfbf4410b5887921f2aded32
2024-11-09T21:23:31,306 INFO  [M:0;d86ff69d6719:33537 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/100c6ca9cfbf4410b5887921f2aded32, entries=2, sequenceid=60, filesize=5.1 K
2024-11-09T21:23:31,307 DEBUG [M:0;d86ff69d6719:33537 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/912fdc37e1224dd0860045ae6b51b278 as hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/912fdc37e1224dd0860045ae6b51b278
2024-11-09T21:23:31,313 INFO  [M:0;d86ff69d6719:33537 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/912fdc37e1224dd0860045ae6b51b278, entries=1, sequenceid=60, filesize=4.9 K
2024-11-09T21:23:31,314 INFO  [M:0;d86ff69d6719:33537 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 546ms, sequenceid=60, compaction requested=false
2024-11-09T21:23:31,316 INFO  [M:0;d86ff69d6719:33537 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:23:31,316 DEBUG [M:0;d86ff69d6719:33537 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731187410768Disabling compacts and flushes for region at 1731187410768Disabling writes for close at 1731187410768Obtaining lock to block concurrent updates at 1731187410769 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731187410769Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23817, getHeapSize=30144, getOffHeapSize=0, getCellsCount=71 at 1731187410769Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731187410770 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731187410770Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731187410786 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731187410786Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731187410797 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731187410813 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731187410813Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731187410824 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731187410839 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731187410839Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731187410849 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731187410863 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731187410863Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d5f7879: reopening flushed file at 1731187411284 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a94460a: reopening flushed file at 1731187411291 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a76ad60: reopening flushed file at 1731187411299 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d75a5ab: reopening flushed file at 1731187411306 (+7 ms)Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 546ms, sequenceid=60, compaction requested=false at 1731187411314 (+8 ms)Writing region close event to WAL at 1731187411316 (+2 ms)Closed at 1731187411316
2024-11-09T21:23:31,316 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:31,316 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:31,316 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:31,317 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:31,317 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:31,318 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34361 is added to blk_1073741888_1071 (size=1045)
2024-11-09T21:23:31,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37129 is added to blk_1073741888_1071 (size=1045)
2024-11-09T21:23:31,319 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-09T21:23:31,319 INFO  [M:0;d86ff69d6719:33537 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-09T21:23:31,319 INFO  [M:0;d86ff69d6719:33537 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33537
2024-11-09T21:23:31,320 INFO  [M:0;d86ff69d6719:33537 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-09T21:23:31,434 INFO  [M:0;d86ff69d6719:33537 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-09T21:23:31,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:23:31,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33537-0x10122c458830000, quorum=127.0.0.1:54077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:23:31,437 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6dbadaea{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:23:31,438 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e48919f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:23:31,438 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:23:31,438 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@213456fa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:23:31,438 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6e6ddf83{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.log.dir/,STOPPED}
2024-11-09T21:23:31,440 WARN  [BP-1331531764-172.17.0.2-1731187359371 heartbeating to localhost/127.0.0.1:42197 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:23:31,440 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:23:31,440 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:23:31,440 WARN  [BP-1331531764-172.17.0.2-1731187359371 heartbeating to localhost/127.0.0.1:42197 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1331531764-172.17.0.2-1731187359371 (Datanode Uuid 2894e6a3-288e-408f-8bd5-b84a909e5593) service to localhost/127.0.0.1:42197
2024-11-09T21:23:31,440 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3c44e655 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1331531764-172.17.0.2-1731187359371:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:32917,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:46751 , LocalHost:localPort d86ff69d6719/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-09T21:23:31,440 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3c44e655 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1331531764-172.17.0.2-1731187359371:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:37129,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1331531764-172.17.0.2-1731187359371
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:31,441 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3c44e655 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1331531764-172.17.0.2-1731187359371:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:32917,null,null], DatanodeInfoWithStorage[127.0.0.1:37129,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1331531764-172.17.0.2-1731187359371:blk_1073741837_1013, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:32917,null,null], DatanodeInfoWithStorage[127.0.0.1:37129,null,null]]
2024-11-09T21:23:31,441 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data3/current/BP-1331531764-172.17.0.2-1731187359371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:23:31,441 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3c44e655 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1331531764-172.17.0.2-1731187359371:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:37129,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1331531764-172.17.0.2-1731187359371
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:31,441 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3c44e655 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1331531764-172.17.0.2-1731187359371:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:32917,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1331531764-172.17.0.2-1731187359371
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:31,441 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data4/current/BP-1331531764-172.17.0.2-1731187359371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:23:31,441 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3c44e655 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1331531764-172.17.0.2-1731187359371:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:37129,null,null], DatanodeInfoWithStorage[127.0.0.1:32917,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1331531764-172.17.0.2-1731187359371:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:37129,null,null], DatanodeInfoWithStorage[127.0.0.1:32917,null,null]]
2024-11-09T21:23:31,442 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:23:31,446 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:31,447 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5e08dd81{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:23:31,448 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@178f342a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:23:31,448 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:23:31,448 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5455501c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:23:31,448 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fe58b15{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.log.dir/,STOPPED}
2024-11-09T21:23:31,449 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:23:31,449 WARN  [BP-1331531764-172.17.0.2-1731187359371 heartbeating to localhost/127.0.0.1:42197 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:23:31,449 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:23:31,449 WARN  [BP-1331531764-172.17.0.2-1731187359371 heartbeating to localhost/127.0.0.1:42197 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1331531764-172.17.0.2-1731187359371 (Datanode Uuid 141d9b99-a758-4387-9668-071f4ea5ccf7) service to localhost/127.0.0.1:42197
2024-11-09T21:23:31,450 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data7/current/BP-1331531764-172.17.0.2-1731187359371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:23:31,450 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/cluster_2d622170-983a-7efe-7dce-061db1c297a5/data/data8/current/BP-1331531764-172.17.0.2-1731187359371 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:23:31,450 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:23:31,456 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@75096fee{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-09T21:23:31,457 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@346c0162{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:23:31,457 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:23:31,457 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3e26ba04{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:23:31,457 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4ac253d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.log.dir/,STOPPED}
2024-11-09T21:23:31,464 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-09T21:23:31,470 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:31,495 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-09T21:23:31,503 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=158 (was 82)
Potentially hanging thread: nioEventLoopGroup-17-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42197 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42197
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:42197
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42197
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42197
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42197 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f6f44bef5c0.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:42197
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42197
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42197 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: nioEventLoopGroup-17-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:35901
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f6f44bef5c0.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f6f44bef5c0.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:35901
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42197 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:42197
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=436 (was 404) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=53 (was 39) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=18229 (was 18627)
2024-11-09T21:23:31,509 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=158, OpenFileDescriptor=436, MaxFileDescriptor=1048576, SystemLoadAverage=53, ProcessCount=11, AvailableMemoryMB=18228
2024-11-09T21:23:31,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-09T21:23:31,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.log.dir so I do NOT create it in target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342
2024-11-09T21:23:31,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/28f9d7d1-b04d-38a6-60bb-afc5ff4cd4a0/hadoop.tmp.dir so I do NOT create it in target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342
2024-11-09T21:23:31,510 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b, deleteOnExit=true
2024-11-09T21:23:31,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-09T21:23:31,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/test.cache.data in system properties and HBase conf
2024-11-09T21:23:31,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.tmp.dir in system properties and HBase conf
2024-11-09T21:23:31,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.log.dir in system properties and HBase conf
2024-11-09T21:23:31,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-09T21:23:31,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-09T21:23:31,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-09T21:23:31,511 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-09T21:23:31,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-09T21:23:31,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-09T21:23:31,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-09T21:23:31,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-09T21:23:31,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-09T21:23:31,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-09T21:23:31,512 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-09T21:23:31,512 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-09T21:23:31,512 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-09T21:23:31,512 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/nfs.dump.dir in system properties and HBase conf
2024-11-09T21:23:31,512 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/java.io.tmpdir in system properties and HBase conf
2024-11-09T21:23:31,512 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-09T21:23:31,512 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-09T21:23:31,512 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-09T21:23:31,524 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-09T21:23:31,805 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:23:31,809 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:23:31,810 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:23:31,810 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:23:31,811 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-09T21:23:31,811 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:23:31,812 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a011456{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:23:31,812 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a6464d2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:23:31,903 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34c1099f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/java.io.tmpdir/jetty-localhost-43981-hadoop-hdfs-3_4_1-tests_jar-_-any-7523064813891490393/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-09T21:23:31,903 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@56e801fe{HTTP/1.1, (http/1.1)}{localhost:43981}
2024-11-09T21:23:31,903 INFO  [Time-limited test {}] server.Server(415): Started @159322ms
2024-11-09T21:23:31,915 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-09T21:23:32,084 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:23:32,087 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:23:32,090 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:23:32,090 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:23:32,090 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-09T21:23:32,091 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@23d454c3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:23:32,091 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7bb583ee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:23:32,187 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@68c19d09{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/java.io.tmpdir/jetty-localhost-45867-hadoop-hdfs-3_4_1-tests_jar-_-any-2670770152415085150/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:23:32,188 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@64434c96{HTTP/1.1, (http/1.1)}{localhost:45867}
2024-11-09T21:23:32,188 INFO  [Time-limited test {}] server.Server(415): Started @159607ms
2024-11-09T21:23:32,188 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:32,189 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:23:32,216 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:23:32,220 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:23:32,221 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:23:32,221 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:23:32,221 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T21:23:32,222 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@50443adc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:23:32,222 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2786a051{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:23:32,316 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@51dac856{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/java.io.tmpdir/jetty-localhost-45927-hadoop-hdfs-3_4_1-tests_jar-_-any-703240634525404498/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:23:32,316 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6bf577b6{HTTP/1.1, (http/1.1)}{localhost:45927}
2024-11-09T21:23:32,316 INFO  [Time-limited test {}] server.Server(415): Started @159735ms
2024-11-09T21:23:32,317 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:23:32,447 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:32,470 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:32,991 WARN  [Thread-1191 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/data/data1/current/BP-289274094-172.17.0.2-1731187411535/current, will proceed with Du for space computation calculation, 
2024-11-09T21:23:32,991 WARN  [Thread-1192 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/data/data2/current/BP-289274094-172.17.0.2-1731187411535/current, will proceed with Du for space computation calculation, 
2024-11-09T21:23:33,009 WARN  [Thread-1155 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:23:33,011 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9faf9318a662e94c with lease ID 0xb8b18762ab377c4f: Processing first storage report for DS-8036443f-aa99-456b-bc73-67ff7f3fc77f from datanode DatanodeRegistration(127.0.0.1:36507, datanodeUuid=5e4f120a-8b81-4cb2-88f9-73f064cc8cf1, infoPort=33711, infoSecurePort=0, ipcPort=41629, storageInfo=lv=-57;cid=testClusterID;nsid=1886735975;c=1731187411535)
2024-11-09T21:23:33,011 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9faf9318a662e94c with lease ID 0xb8b18762ab377c4f: from storage DS-8036443f-aa99-456b-bc73-67ff7f3fc77f node DatanodeRegistration(127.0.0.1:36507, datanodeUuid=5e4f120a-8b81-4cb2-88f9-73f064cc8cf1, infoPort=33711, infoSecurePort=0, ipcPort=41629, storageInfo=lv=-57;cid=testClusterID;nsid=1886735975;c=1731187411535), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:23:33,011 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9faf9318a662e94c with lease ID 0xb8b18762ab377c4f: Processing first storage report for DS-82ccbe06-7608-4295-b707-479f1f7198ce from datanode DatanodeRegistration(127.0.0.1:36507, datanodeUuid=5e4f120a-8b81-4cb2-88f9-73f064cc8cf1, infoPort=33711, infoSecurePort=0, ipcPort=41629, storageInfo=lv=-57;cid=testClusterID;nsid=1886735975;c=1731187411535)
2024-11-09T21:23:33,011 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9faf9318a662e94c with lease ID 0xb8b18762ab377c4f: from storage DS-82ccbe06-7608-4295-b707-479f1f7198ce node DatanodeRegistration(127.0.0.1:36507, datanodeUuid=5e4f120a-8b81-4cb2-88f9-73f064cc8cf1, infoPort=33711, infoSecurePort=0, ipcPort=41629, storageInfo=lv=-57;cid=testClusterID;nsid=1886735975;c=1731187411535), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:23:33,102 WARN  [Thread-1202 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/data/data3/current/BP-289274094-172.17.0.2-1731187411535/current, will proceed with Du for space computation calculation, 
2024-11-09T21:23:33,103 WARN  [Thread-1203 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/data/data4/current/BP-289274094-172.17.0.2-1731187411535/current, will proceed with Du for space computation calculation, 
2024-11-09T21:23:33,125 WARN  [Thread-1178 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:23:33,127 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x96af6425c0fdb31d with lease ID 0xb8b18762ab377c50: Processing first storage report for DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2 from datanode DatanodeRegistration(127.0.0.1:46499, datanodeUuid=e7072753-4f01-4a69-9c7c-d5479a6d7d5d, infoPort=45599, infoSecurePort=0, ipcPort=44935, storageInfo=lv=-57;cid=testClusterID;nsid=1886735975;c=1731187411535)
2024-11-09T21:23:33,127 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x96af6425c0fdb31d with lease ID 0xb8b18762ab377c50: from storage DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2 node DatanodeRegistration(127.0.0.1:46499, datanodeUuid=e7072753-4f01-4a69-9c7c-d5479a6d7d5d, infoPort=45599, infoSecurePort=0, ipcPort=44935, storageInfo=lv=-57;cid=testClusterID;nsid=1886735975;c=1731187411535), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:23:33,127 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x96af6425c0fdb31d with lease ID 0xb8b18762ab377c50: Processing first storage report for DS-4240ffac-7920-4166-879f-9db6efc5937c from datanode DatanodeRegistration(127.0.0.1:46499, datanodeUuid=e7072753-4f01-4a69-9c7c-d5479a6d7d5d, infoPort=45599, infoSecurePort=0, ipcPort=44935, storageInfo=lv=-57;cid=testClusterID;nsid=1886735975;c=1731187411535)
2024-11-09T21:23:33,127 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x96af6425c0fdb31d with lease ID 0xb8b18762ab377c50: from storage DS-4240ffac-7920-4166-879f-9db6efc5937c node DatanodeRegistration(127.0.0.1:46499, datanodeUuid=e7072753-4f01-4a69-9c7c-d5479a6d7d5d, infoPort=45599, infoSecurePort=0, ipcPort=44935, storageInfo=lv=-57;cid=testClusterID;nsid=1886735975;c=1731187411535), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:23:33,171 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342
2024-11-09T21:23:33,175 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/zookeeper_0, clientPort=62742, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-09T21:23:33,176 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62742
2024-11-09T21:23:33,176 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:23:33,177 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:23:33,184 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741825_1001 (size=7)
2024-11-09T21:23:33,185 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741825_1001 (size=7)
2024-11-09T21:23:33,186 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2 with version=8
2024-11-09T21:23:33,186 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/hbase-staging
2024-11-09T21:23:33,188 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d86ff69d6719:0 server-side Connection retries=45
2024-11-09T21:23:33,188 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:23:33,188 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T21:23:33,188 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T21:23:33,188 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:23:33,188 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T21:23:33,188 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-09T21:23:33,189 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T21:23:33,189 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:33,189 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35765
2024-11-09T21:23:33,191 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35765 connecting to ZooKeeper ensemble=127.0.0.1:62742
2024-11-09T21:23:33,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:357650x0, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T21:23:33,242 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35765-0x10122c5237a0000 connected
2024-11-09T21:23:33,320 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:23:33,322 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:23:33,325 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:23:33,325 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2, hbase.cluster.distributed=false
2024-11-09T21:23:33,329 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T21:23:33,330 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35765
2024-11-09T21:23:33,330 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35765
2024-11-09T21:23:33,330 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35765
2024-11-09T21:23:33,331 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35765
2024-11-09T21:23:33,331 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35765
2024-11-09T21:23:33,347 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d86ff69d6719:0 server-side Connection retries=45
2024-11-09T21:23:33,347 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:23:33,347 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T21:23:33,347 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T21:23:33,347 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:23:33,347 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T21:23:33,347 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-09T21:23:33,347 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T21:23:33,348 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45987
2024-11-09T21:23:33,350 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45987 connecting to ZooKeeper ensemble=127.0.0.1:62742
2024-11-09T21:23:33,350 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:23:33,351 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:23:33,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:459870x0, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T21:23:33,362 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:459870x0, quorum=127.0.0.1:62742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:23:33,362 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45987-0x10122c5237a0001 connected
2024-11-09T21:23:33,362 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-09T21:23:33,363 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-09T21:23:33,364 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-09T21:23:33,365 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T21:23:33,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45987
2024-11-09T21:23:33,368 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45987
2024-11-09T21:23:33,370 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45987
2024-11-09T21:23:33,373 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45987
2024-11-09T21:23:33,373 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45987
2024-11-09T21:23:33,385 DEBUG [M:0;d86ff69d6719:35765 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d86ff69d6719:35765
2024-11-09T21:23:33,385 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d86ff69d6719,35765,1731187413188
2024-11-09T21:23:33,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:23:33,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:23:33,395 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d86ff69d6719,35765,1731187413188
2024-11-09T21:23:33,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:33,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-09T21:23:33,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:33,403 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-09T21:23:33,404 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d86ff69d6719,35765,1731187413188 from backup master directory
2024-11-09T21:23:33,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:23:33,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d86ff69d6719,35765,1731187413188
2024-11-09T21:23:33,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:23:33,411 WARN  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T21:23:33,411 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d86ff69d6719,35765,1731187413188
2024-11-09T21:23:33,416 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/hbase.id] with ID: 62f4f9c5-72e4-46d8-aa03-3e37eb3f63c8
2024-11-09T21:23:33,416 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/.tmp/hbase.id
2024-11-09T21:23:33,422 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741826_1002 (size=42)
2024-11-09T21:23:33,422 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741826_1002 (size=42)
2024-11-09T21:23:33,422 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/.tmp/hbase.id]:[hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/hbase.id]
2024-11-09T21:23:33,433 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:23:33,433 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-09T21:23:33,435 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-09T21:23:33,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:33,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:33,448 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:33,451 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741827_1003 (size=196)
2024-11-09T21:23:33,451 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741827_1003 (size=196)
2024-11-09T21:23:33,452 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-09T21:23:33,453 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-09T21:23:33,453 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:23:33,460 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741828_1004 (size=1189)
2024-11-09T21:23:33,460 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741828_1004 (size=1189)
2024-11-09T21:23:33,461 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store
2024-11-09T21:23:33,467 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741829_1005 (size=34)
2024-11-09T21:23:33,467 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741829_1005 (size=34)
2024-11-09T21:23:33,468 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:23:33,468 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-09T21:23:33,468 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:23:33,468 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:23:33,468 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-09T21:23:33,468 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:23:33,468 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:23:33,468 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731187413468Disabling compacts and flushes for region at 1731187413468Disabling writes for close at 1731187413468Writing region close event to WAL at 1731187413468Closed at 1731187413468
2024-11-09T21:23:33,469 WARN  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/.initializing
2024-11-09T21:23:33,469 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/WALs/d86ff69d6719,35765,1731187413188
2024-11-09T21:23:33,471 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:33,471 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C35765%2C1731187413188, suffix=, logDir=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/WALs/d86ff69d6719,35765,1731187413188, archiveDir=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/oldWALs, maxLogs=10
2024-11-09T21:23:33,472 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C35765%2C1731187413188.1731187413472
2024-11-09T21:23:33,477 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/WALs/d86ff69d6719,35765,1731187413188/d86ff69d6719%2C35765%2C1731187413188.1731187413472
2024-11-09T21:23:33,477 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33711:33711),(127.0.0.1/127.0.0.1:45599:45599)]
2024-11-09T21:23:33,478 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:23:33,478 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:23:33,478 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:23:33,478 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:23:33,479 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:23:33,481 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-09T21:23:33,481 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:23:33,482 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:23:33,482 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:23:33,483 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-09T21:23:33,483 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:23:33,484 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:23:33,484 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:23:33,485 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-09T21:23:33,485 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:23:33,486 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:23:33,486 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:23:33,487 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-09T21:23:33,487 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:23:33,488 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:23:33,488 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:23:33,489 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:23:33,489 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:23:33,491 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:23:33,491 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:23:33,491 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-09T21:23:33,492 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:23:33,494 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:23:33,495 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=774783, jitterRate=-0.014812633395195007}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-09T21:23:33,495 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731187413478Initializing all the Stores at 1731187413479 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187413479Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187413479Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187413479Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187413479Cleaning up temporary data from old regions at 1731187413491 (+12 ms)Region opened successfully at 1731187413495 (+4 ms)
2024-11-09T21:23:33,496 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-09T21:23:33,499 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c48ec67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0
2024-11-09T21:23:33,500 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-09T21:23:33,501 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-09T21:23:33,501 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-09T21:23:33,501 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-09T21:23:33,501 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-09T21:23:33,502 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-09T21:23:33,502 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-09T21:23:33,504 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-09T21:23:33,505 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-09T21:23:33,520 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-09T21:23:33,520 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-09T21:23:33,521 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-09T21:23:33,528 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-09T21:23:33,528 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-09T21:23:33,529 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-09T21:23:33,536 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-09T21:23:33,537 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-09T21:23:33,545 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-09T21:23:33,547 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-09T21:23:33,553 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-09T21:23:33,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-09T21:23:33,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-09T21:23:33,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:33,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:33,562 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d86ff69d6719,35765,1731187413188, sessionid=0x10122c5237a0000, setting cluster-up flag (Was=false)
2024-11-09T21:23:33,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:33,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:33,603 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-09T21:23:33,604 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d86ff69d6719,35765,1731187413188
2024-11-09T21:23:33,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:33,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:33,645 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-09T21:23:33,646 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d86ff69d6719,35765,1731187413188
2024-11-09T21:23:33,647 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-09T21:23:33,649 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-09T21:23:33,649 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-09T21:23:33,649 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-09T21:23:33,649 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d86ff69d6719,35765,1731187413188
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-09T21:23:33,650 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:23:33,651 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:23:33,651 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:23:33,651 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:23:33,651 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d86ff69d6719:0, corePoolSize=10, maxPoolSize=10
2024-11-09T21:23:33,651 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:23:33,651 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2
2024-11-09T21:23:33,651 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:23:33,652 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731187443652
2024-11-09T21:23:33,652 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-09T21:23:33,652 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-09T21:23:33,652 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-09T21:23:33,652 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-09T21:23:33,653 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-09T21:23:33,653 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-09T21:23:33,653 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:33,653 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:23:33,653 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-09T21:23:33,653 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-09T21:23:33,653 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-09T21:23:33,653 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-09T21:23:33,654 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-09T21:23:33,654 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-09T21:23:33,654 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187413654,5,FailOnTimeoutGroup]
2024-11-09T21:23:33,654 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187413654,5,FailOnTimeoutGroup]
2024-11-09T21:23:33,654 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:33,654 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:23:33,654 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-09T21:23:33,654 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:33,654 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:33,654 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-09T21:23:33,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741831_1007 (size=1321)
2024-11-09T21:23:33,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741831_1007 (size=1321)
2024-11-09T21:23:33,664 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-09T21:23:33,664 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2
2024-11-09T21:23:33,670 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741832_1008 (size=32)
2024-11-09T21:23:33,670 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741832_1008 (size=32)
2024-11-09T21:23:33,670 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:23:33,671 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-09T21:23:33,673 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-09T21:23:33,673 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:23:33,674 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:23:33,674 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-09T21:23:33,675 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer(746): ClusterId : 62f4f9c5-72e4-46d8-aa03-3e37eb3f63c8
2024-11-09T21:23:33,675 DEBUG [RS:0;d86ff69d6719:45987 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-09T21:23:33,675 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-09T21:23:33,676 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:23:33,676 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:23:33,676 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-09T21:23:33,677 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-09T21:23:33,677 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:23:33,678 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:23:33,678 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-09T21:23:33,679 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-09T21:23:33,679 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:23:33,680 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:23:33,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-09T21:23:33,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740
2024-11-09T21:23:33,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740
2024-11-09T21:23:33,681 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-09T21:23:33,681 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-09T21:23:33,682 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-09T21:23:33,683 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-09T21:23:33,684 DEBUG [RS:0;d86ff69d6719:45987 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-09T21:23:33,684 DEBUG [RS:0;d86ff69d6719:45987 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-09T21:23:33,685 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:23:33,685 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=780765, jitterRate=-0.007206946611404419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-09T21:23:33,686 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731187413670Initializing all the Stores at 1731187413671 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187413671Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187413671Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187413671Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187413671Cleaning up temporary data from old regions at 1731187413682 (+11 ms)Region opened successfully at 1731187413686 (+4 ms)
2024-11-09T21:23:33,686 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-09T21:23:33,686 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-09T21:23:33,686 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-09T21:23:33,686 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-09T21:23:33,686 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-09T21:23:33,686 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-09T21:23:33,686 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731187413686Disabling compacts and flushes for region at 1731187413686Disabling writes for close at 1731187413686Writing region close event to WAL at 1731187413686Closed at 1731187413686
2024-11-09T21:23:33,687 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:23:33,687 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-09T21:23:33,688 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-09T21:23:33,689 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-09T21:23:33,689 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-09T21:23:33,695 DEBUG [RS:0;d86ff69d6719:45987 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-09T21:23:33,696 DEBUG [RS:0;d86ff69d6719:45987 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69088f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0
2024-11-09T21:23:33,707 DEBUG [RS:0;d86ff69d6719:45987 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d86ff69d6719:45987
2024-11-09T21:23:33,707 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-09T21:23:33,707 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-09T21:23:33,707 DEBUG [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-09T21:23:33,708 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer(2659): reportForDuty to master=d86ff69d6719,35765,1731187413188 with port=45987, startcode=1731187413346
2024-11-09T21:23:33,708 DEBUG [RS:0;d86ff69d6719:45987 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-09T21:23:33,710 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40865, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-11-09T21:23:33,710 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35765 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d86ff69d6719,45987,1731187413346
2024-11-09T21:23:33,710 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35765 {}] master.ServerManager(517): Registering regionserver=d86ff69d6719,45987,1731187413346
2024-11-09T21:23:33,712 DEBUG [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2
2024-11-09T21:23:33,712 DEBUG [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38013
2024-11-09T21:23:33,712 DEBUG [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-09T21:23:33,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T21:23:33,720 DEBUG [RS:0;d86ff69d6719:45987 {}] zookeeper.ZKUtil(111): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d86ff69d6719,45987,1731187413346
2024-11-09T21:23:33,720 WARN  [RS:0;d86ff69d6719:45987 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T21:23:33,720 INFO  [RS:0;d86ff69d6719:45987 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:23:33,721 DEBUG [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346
2024-11-09T21:23:33,721 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d86ff69d6719,45987,1731187413346]
2024-11-09T21:23:33,724 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-09T21:23:33,727 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-09T21:23:33,727 INFO  [RS:0;d86ff69d6719:45987 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-09T21:23:33,727 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:33,727 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-09T21:23:33,728 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-09T21:23:33,728 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:33,728 DEBUG [RS:0;d86ff69d6719:45987 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:23:33,728 DEBUG [RS:0;d86ff69d6719:45987 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:23:33,729 DEBUG [RS:0;d86ff69d6719:45987 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:23:33,729 DEBUG [RS:0;d86ff69d6719:45987 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:23:33,729 DEBUG [RS:0;d86ff69d6719:45987 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:23:33,729 DEBUG [RS:0;d86ff69d6719:45987 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2
2024-11-09T21:23:33,729 DEBUG [RS:0;d86ff69d6719:45987 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:23:33,729 DEBUG [RS:0;d86ff69d6719:45987 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:23:33,729 DEBUG [RS:0;d86ff69d6719:45987 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:23:33,729 DEBUG [RS:0;d86ff69d6719:45987 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:23:33,729 DEBUG [RS:0;d86ff69d6719:45987 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:23:33,729 DEBUG [RS:0;d86ff69d6719:45987 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:23:33,729 DEBUG [RS:0;d86ff69d6719:45987 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3
2024-11-09T21:23:33,729 DEBUG [RS:0;d86ff69d6719:45987 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3
2024-11-09T21:23:33,730 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:33,730 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:33,731 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:33,731 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:33,731 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:33,731 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,45987,1731187413346-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T21:23:33,744 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-09T21:23:33,744 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,45987,1731187413346-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:33,745 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:33,745 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.Replication(171): d86ff69d6719,45987,1731187413346 started
2024-11-09T21:23:33,758 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:33,758 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer(1482): Serving as d86ff69d6719,45987,1731187413346, RpcServer on d86ff69d6719/172.17.0.2:45987, sessionid=0x10122c5237a0001
2024-11-09T21:23:33,758 DEBUG [RS:0;d86ff69d6719:45987 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-09T21:23:33,758 DEBUG [RS:0;d86ff69d6719:45987 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d86ff69d6719,45987,1731187413346
2024-11-09T21:23:33,758 DEBUG [RS:0;d86ff69d6719:45987 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,45987,1731187413346'
2024-11-09T21:23:33,758 DEBUG [RS:0;d86ff69d6719:45987 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-09T21:23:33,759 DEBUG [RS:0;d86ff69d6719:45987 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-09T21:23:33,759 DEBUG [RS:0;d86ff69d6719:45987 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-09T21:23:33,759 DEBUG [RS:0;d86ff69d6719:45987 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-09T21:23:33,759 DEBUG [RS:0;d86ff69d6719:45987 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d86ff69d6719,45987,1731187413346
2024-11-09T21:23:33,759 DEBUG [RS:0;d86ff69d6719:45987 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,45987,1731187413346'
2024-11-09T21:23:33,759 DEBUG [RS:0;d86ff69d6719:45987 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-09T21:23:33,760 DEBUG [RS:0;d86ff69d6719:45987 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-09T21:23:33,760 DEBUG [RS:0;d86ff69d6719:45987 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-09T21:23:33,760 INFO  [RS:0;d86ff69d6719:45987 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-09T21:23:33,760 INFO  [RS:0;d86ff69d6719:45987 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-09T21:23:33,840 WARN  [d86ff69d6719:35765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-09T21:23:33,863 INFO  [RS:0;d86ff69d6719:45987 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C45987%2C1731187413346, suffix=, logDir=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346, archiveDir=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/oldWALs, maxLogs=32
2024-11-09T21:23:33,863 INFO  [RS:0;d86ff69d6719:45987 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C45987%2C1731187413346.1731187413863
2024-11-09T21:23:33,870 INFO  [RS:0;d86ff69d6719:45987 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187413863
2024-11-09T21:23:33,871 DEBUG [RS:0;d86ff69d6719:45987 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33711:33711),(127.0.0.1/127.0.0.1:45599:45599)]
2024-11-09T21:23:34,090 DEBUG [d86ff69d6719:35765 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-09T21:23:34,091 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d86ff69d6719,45987,1731187413346
2024-11-09T21:23:34,095 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d86ff69d6719,45987,1731187413346, state=OPENING
2024-11-09T21:23:34,144 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-09T21:23:34,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:34,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:34,153 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-09T21:23:34,153 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:23:34,153 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:23:34,153 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d86ff69d6719,45987,1731187413346}]
2024-11-09T21:23:34,189 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:34,307 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-09T21:23:34,310 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53991, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-09T21:23:34,317 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-09T21:23:34,318 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:23:34,320 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C45987%2C1731187413346.meta, suffix=.meta, logDir=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346, archiveDir=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/oldWALs, maxLogs=32
2024-11-09T21:23:34,320 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C45987%2C1731187413346.meta.1731187414320.meta
2024-11-09T21:23:34,325 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.meta.1731187414320.meta
2024-11-09T21:23:34,326 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45599:45599),(127.0.0.1/127.0.0.1:33711:33711)]
2024-11-09T21:23:34,327 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:23:34,327 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-09T21:23:34,327 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-09T21:23:34,327 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-09T21:23:34,327 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-09T21:23:34,327 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:23:34,327 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-09T21:23:34,327 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-09T21:23:34,329 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-09T21:23:34,330 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-09T21:23:34,330 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:23:34,330 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:23:34,330 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-09T21:23:34,331 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-09T21:23:34,331 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:23:34,332 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:23:34,332 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-09T21:23:34,332 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-09T21:23:34,332 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:23:34,333 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:23:34,333 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-09T21:23:34,334 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-09T21:23:34,334 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:23:34,334 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:23:34,334 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-09T21:23:34,335 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740
2024-11-09T21:23:34,336 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740
2024-11-09T21:23:34,338 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-09T21:23:34,338 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-09T21:23:34,338 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-09T21:23:34,340 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-09T21:23:34,341 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=731137, jitterRate=-0.0703112781047821}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-09T21:23:34,341 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-09T21:23:34,341 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731187414328Writing region info on filesystem at 1731187414328Initializing all the Stores at 1731187414328Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187414328Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187414329 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187414329Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187414329Cleaning up temporary data from old regions at 1731187414338 (+9 ms)Running coprocessor post-open hooks at 1731187414341 (+3 ms)Region opened successfully at 1731187414341
2024-11-09T21:23:34,342 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731187414306
2024-11-09T21:23:34,345 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-09T21:23:34,345 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-09T21:23:34,346 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,45987,1731187413346
2024-11-09T21:23:34,347 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d86ff69d6719,45987,1731187413346, state=OPEN
2024-11-09T21:23:34,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-09T21:23:34,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-09T21:23:34,382 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d86ff69d6719,45987,1731187413346
2024-11-09T21:23:34,382 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:23:34,382 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:23:34,387 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-09T21:23:34,387 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d86ff69d6719,45987,1731187413346 in 229 msec
2024-11-09T21:23:34,392 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-09T21:23:34,392 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 700 msec
2024-11-09T21:23:34,393 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:23:34,393 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-09T21:23:34,395 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-09T21:23:34,395 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,45987,1731187413346, seqNum=-1]
2024-11-09T21:23:34,395 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T21:23:34,397 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39643, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T21:23:34,405 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 755 msec
2024-11-09T21:23:34,405 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731187414405, completionTime=-1
2024-11-09T21:23:34,405 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-09T21:23:34,405 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-09T21:23:34,408 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-09T21:23:34,408 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731187474408
2024-11-09T21:23:34,408 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731187534408
2024-11-09T21:23:34,408 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-09T21:23:34,408 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,35765,1731187413188-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:34,408 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,35765,1731187413188-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:34,408 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,35765,1731187413188-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:34,408 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d86ff69d6719:35765, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:34,409 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:34,409 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:34,411 DEBUG [master/d86ff69d6719:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-09T21:23:34,413 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.002sec
2024-11-09T21:23:34,413 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-09T21:23:34,413 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-09T21:23:34,413 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-09T21:23:34,413 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-09T21:23:34,413 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-09T21:23:34,413 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,35765,1731187413188-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T21:23:34,413 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,35765,1731187413188-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-09T21:23:34,417 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-09T21:23:34,417 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-09T21:23:34,417 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,35765,1731187413188-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:23:34,448 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:34,472 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:34,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ca8564b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:23:34,476 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d86ff69d6719,35765,-1 for getting cluster id
2024-11-09T21:23:34,476 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-09T21:23:34,477 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62f4f9c5-72e4-46d8-aa03-3e37eb3f63c8'
2024-11-09T21:23:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-09T21:23:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62f4f9c5-72e4-46d8-aa03-3e37eb3f63c8"

2024-11-09T21:23:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79c24a43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:23:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d86ff69d6719,35765,-1]
2024-11-09T21:23:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-09T21:23:34,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:23:34,480 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-09T21:23:34,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45604664, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:23:34,481 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-09T21:23:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,45987,1731187413346, seqNum=-1]
2024-11-09T21:23:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T21:23:34,484 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T21:23:34,486 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d86ff69d6719,35765,1731187413188
2024-11-09T21:23:34,486 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:23:34,488 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-09T21:23:34,488 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-11-09T21:23:34,488 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-11-09T21:23:34,488 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-09T21:23:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is d86ff69d6719,35765,1731187413188
2024-11-09T21:23:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7f4a93b1
2024-11-09T21:23:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-09T21:23:34,491 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-09T21:23:34,492 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35765 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-09T21:23:34,492 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35765 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-09T21:23:34,492 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35765 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-09T21:23:34,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35765 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-11-09T21:23:34,495 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-11-09T21:23:34,495 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:23:34,495 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35765 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-11-09T21:23:34,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-09T21:23:34,496 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-09T21:23:34,503 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741835_1011 (size=395)
2024-11-09T21:23:34,503 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741835_1011 (size=395)
2024-11-09T21:23:34,504 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 595d6bfca74458389c5bf371f7c86a09, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2
2024-11-09T21:23:34,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46499 is added to blk_1073741836_1012 (size=78)
2024-11-09T21:23:34,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741836_1012 (size=78)
2024-11-09T21:23:34,511 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:23:34,511 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 595d6bfca74458389c5bf371f7c86a09, disabling compactions & flushes
2024-11-09T21:23:34,511 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.
2024-11-09T21:23:34,511 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.
2024-11-09T21:23:34,511 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09. after waiting 0 ms
2024-11-09T21:23:34,511 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.
2024-11-09T21:23:34,511 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.
2024-11-09T21:23:34,511 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 595d6bfca74458389c5bf371f7c86a09:
Waiting for close lock at 1731187414511Disabling compacts and flushes for region at 1731187414511Disabling writes for close at 1731187414511Writing region close event to WAL at 1731187414511Closed at 1731187414511
2024-11-09T21:23:34,513 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-11-09T21:23:34,513 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731187414513"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731187414513"}]},"ts":"1731187414513"}
2024-11-09T21:23:34,515 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-09T21:23:34,516 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-09T21:23:34,517 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187414516"}]},"ts":"1731187414516"}
2024-11-09T21:23:34,519 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-11-09T21:23:34,519 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=595d6bfca74458389c5bf371f7c86a09, ASSIGN}]
2024-11-09T21:23:34,520 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=595d6bfca74458389c5bf371f7c86a09, ASSIGN
2024-11-09T21:23:34,521 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=595d6bfca74458389c5bf371f7c86a09, ASSIGN; state=OFFLINE, location=d86ff69d6719,45987,1731187413346; forceNewPlan=false, retain=false
2024-11-09T21:23:34,672 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=595d6bfca74458389c5bf371f7c86a09, regionState=OPENING, regionLocation=d86ff69d6719,45987,1731187413346
2024-11-09T21:23:34,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=595d6bfca74458389c5bf371f7c86a09, ASSIGN because future has completed
2024-11-09T21:23:34,677 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 595d6bfca74458389c5bf371f7c86a09, server=d86ff69d6719,45987,1731187413346}]
2024-11-09T21:23:34,843 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.
2024-11-09T21:23:34,844 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 595d6bfca74458389c5bf371f7c86a09, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:23:34,844 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 595d6bfca74458389c5bf371f7c86a09
2024-11-09T21:23:34,845 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:23:34,845 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 595d6bfca74458389c5bf371f7c86a09
2024-11-09T21:23:34,845 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 595d6bfca74458389c5bf371f7c86a09
2024-11-09T21:23:34,848 INFO  [StoreOpener-595d6bfca74458389c5bf371f7c86a09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 595d6bfca74458389c5bf371f7c86a09 
2024-11-09T21:23:34,850 INFO  [StoreOpener-595d6bfca74458389c5bf371f7c86a09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 595d6bfca74458389c5bf371f7c86a09 columnFamilyName info
2024-11-09T21:23:34,850 DEBUG [StoreOpener-595d6bfca74458389c5bf371f7c86a09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:23:34,850 INFO  [StoreOpener-595d6bfca74458389c5bf371f7c86a09-1 {}] regionserver.HStore(327): Store=595d6bfca74458389c5bf371f7c86a09/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:23:34,851 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 595d6bfca74458389c5bf371f7c86a09
2024-11-09T21:23:34,852 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/default/TestLogRolling-testLogRollOnPipelineRestart/595d6bfca74458389c5bf371f7c86a09
2024-11-09T21:23:34,853 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/default/TestLogRolling-testLogRollOnPipelineRestart/595d6bfca74458389c5bf371f7c86a09
2024-11-09T21:23:34,853 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 595d6bfca74458389c5bf371f7c86a09
2024-11-09T21:23:34,853 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 595d6bfca74458389c5bf371f7c86a09
2024-11-09T21:23:34,856 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 595d6bfca74458389c5bf371f7c86a09
2024-11-09T21:23:34,858 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/default/TestLogRolling-testLogRollOnPipelineRestart/595d6bfca74458389c5bf371f7c86a09/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:23:34,859 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 595d6bfca74458389c5bf371f7c86a09; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=860504, jitterRate=0.09418852627277374}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-09T21:23:34,859 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 595d6bfca74458389c5bf371f7c86a09
2024-11-09T21:23:34,860 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 595d6bfca74458389c5bf371f7c86a09:
Running coprocessor pre-open hook at 1731187414845Writing region info on filesystem at 1731187414845Initializing all the Stores at 1731187414847 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187414847Cleaning up temporary data from old regions at 1731187414853 (+6 ms)Running coprocessor post-open hooks at 1731187414859 (+6 ms)Region opened successfully at 1731187414860 (+1 ms)
2024-11-09T21:23:34,861 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09., pid=6, masterSystemTime=1731187414833
2024-11-09T21:23:34,864 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.
2024-11-09T21:23:34,864 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.
2024-11-09T21:23:34,865 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=595d6bfca74458389c5bf371f7c86a09, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,45987,1731187413346
2024-11-09T21:23:34,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 595d6bfca74458389c5bf371f7c86a09, server=d86ff69d6719,45987,1731187413346 because future has completed
2024-11-09T21:23:34,871 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-09T21:23:34,871 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 595d6bfca74458389c5bf371f7c86a09, server=d86ff69d6719,45987,1731187413346 in 192 msec
2024-11-09T21:23:34,874 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-09T21:23:34,874 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=595d6bfca74458389c5bf371f7c86a09, ASSIGN in 352 msec
2024-11-09T21:23:34,875 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-09T21:23:34,875 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187414875"}]},"ts":"1731187414875"}
2024-11-09T21:23:34,877 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-11-09T21:23:34,878 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-11-09T21:23:34,880 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 386 msec
2024-11-09T21:23:35,067 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-09T21:23:35,087 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:35,087 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:35,087 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:35,088 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:35,088 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:35,088 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:35,091 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:35,091 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:35,091 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:35,094 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:35,190 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:35,449 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:35,472 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:36,191 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:36,450 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:36,474 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:37,192 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:37,452 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:37,475 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:38,193 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:38,453 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:38,476 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:39,194 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:39,454 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:39,477 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:39,563 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-09T21:23:39,564 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-09T21:23:39,566 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-09T21:23:39,566 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-11-09T21:23:39,567 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-09T21:23:39,567 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-09T21:23:39,725 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-09T21:23:39,726 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-11-09T21:23:40,196 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:40,349 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-09T21:23:40,371 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:40,372 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:40,372 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:40,372 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:40,372 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:40,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:40,376 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:40,376 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:40,377 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:40,379 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:23:40,455 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:40,478 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:41,196 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:41,456 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:41,479 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:42,198 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:42,457 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:42,480 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:43,199 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:43,459 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:43,482 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:44,201 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:44,460 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:44,483 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-09T21:23:44,576 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-11-09T21:23:44,576 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-11-09T21:23:44,582 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-11-09T21:23:44,582 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.
2024-11-09T21:23:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09., hostname=d86ff69d6719,45987,1731187413346, seqNum=2]
2024-11-09T21:23:45,202 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:45,461 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:45,484 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:46,204 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:46,463 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:46,485 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:46,593 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187413863
2024-11-09T21:23:46,594 WARN  [ResponseProcessor for block BP-289274094-172.17.0.2-1731187411535:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-289274094-172.17.0.2-1731187411535:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:46,595 WARN  [ResponseProcessor for block BP-289274094-172.17.0.2-1731187411535:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-289274094-172.17.0.2-1731187411535:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-289274094-172.17.0.2-1731187411535:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:46499,DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:46,595 WARN  [ResponseProcessor for block BP-289274094-172.17.0.2-1731187411535:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-289274094-172.17.0.2-1731187411535:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-289274094-172.17.0.2-1731187411535:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:46499,DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:46,596 WARN  [DataStreamer for file /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/WALs/d86ff69d6719,35765,1731187413188/d86ff69d6719%2C35765%2C1731187413188.1731187413472 block BP-289274094-172.17.0.2-1731187411535:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-289274094-172.17.0.2-1731187411535:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36507,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK], DatanodeInfoWithStorage[127.0.0.1:46499,DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46499,DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2,DISK]) is bad.
2024-11-09T21:23:46,596 WARN  [DataStreamer for file /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.meta.1731187414320.meta block BP-289274094-172.17.0.2-1731187411535:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-289274094-172.17.0.2-1731187411535:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46499,DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2,DISK], DatanodeInfoWithStorage[127.0.0.1:36507,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46499,DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2,DISK]) is bad.
2024-11-09T21:23:46,596 WARN  [DataStreamer for file /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187413863 block BP-289274094-172.17.0.2-1731187411535:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-289274094-172.17.0.2-1731187411535:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36507,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK], DatanodeInfoWithStorage[127.0.0.1:46499,DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46499,DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2,DISK]) is bad.
2024-11-09T21:23:46,596 WARN  [PacketResponder: BP-289274094-172.17.0.2-1731187411535:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:46499] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:46,596 WARN  [PacketResponder: BP-289274094-172.17.0.2-1731187411535:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:46499] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:46,598 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1943374404_22 at /127.0.0.1:37628 [Receiving block BP-289274094-172.17.0.2-1731187411535:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46499:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37628 dst: /127.0.0.1:46499
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:46,599 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1943374404_22 at /127.0.0.1:39536 [Receiving block BP-289274094-172.17.0.2-1731187411535:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:36507:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39536 dst: /127.0.0.1:36507
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:46,599 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1133754701_22 at /127.0.0.1:39498 [Receiving block BP-289274094-172.17.0.2-1731187411535:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:36507:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39498 dst: /127.0.0.1:36507
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:46,600 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1943374404_22 at /127.0.0.1:39524 [Receiving block BP-289274094-172.17.0.2-1731187411535:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:36507:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39524 dst: /127.0.0.1:36507
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:46,600 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1943374404_22 at /127.0.0.1:37624 [Receiving block BP-289274094-172.17.0.2-1731187411535:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46499:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37624 dst: /127.0.0.1:46499
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:46,600 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1133754701_22 at /127.0.0.1:37604 [Receiving block BP-289274094-172.17.0.2-1731187411535:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46499:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37604 dst: /127.0.0.1:46499
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:46,700 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@51dac856{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:23:46,701 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6bf577b6{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:23:46,701 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:23:46,701 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2786a051{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:23:46,701 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@50443adc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.log.dir/,STOPPED}
2024-11-09T21:23:46,704 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:23:46,704 WARN  [BP-289274094-172.17.0.2-1731187411535 heartbeating to localhost/127.0.0.1:38013 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:23:46,704 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:23:46,704 WARN  [BP-289274094-172.17.0.2-1731187411535 heartbeating to localhost/127.0.0.1:38013 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-289274094-172.17.0.2-1731187411535 (Datanode Uuid e7072753-4f01-4a69-9c7c-d5479a6d7d5d) service to localhost/127.0.0.1:38013
2024-11-09T21:23:46,705 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/data/data3/current/BP-289274094-172.17.0.2-1731187411535 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:23:46,706 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/data/data4/current/BP-289274094-172.17.0.2-1731187411535 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:23:46,706 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:23:46,713 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:23:46,716 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:23:46,717 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:23:46,717 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:23:46,717 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T21:23:46,719 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a40ff76{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:23:46,720 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4022a798{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:23:46,814 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5a362569{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/java.io.tmpdir/jetty-localhost-38083-hadoop-hdfs-3_4_1-tests_jar-_-any-13750367850540467740/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:23:46,814 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@49f94f8{HTTP/1.1, (http/1.1)}{localhost:38083}
2024-11-09T21:23:46,814 INFO  [Time-limited test {}] server.Server(415): Started @174233ms
2024-11-09T21:23:46,815 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:23:46,835 WARN  [ResponseProcessor for block BP-289274094-172.17.0.2-1731187411535:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-289274094-172.17.0.2-1731187411535:blk_1073741830_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:46,835 WARN  [ResponseProcessor for block BP-289274094-172.17.0.2-1731187411535:blk_1073741834_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-289274094-172.17.0.2-1731187411535:blk_1073741834_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:46,835 WARN  [ResponseProcessor for block BP-289274094-172.17.0.2-1731187411535:blk_1073741833_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-289274094-172.17.0.2-1731187411535:blk_1073741833_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:46,836 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1943374404_22 at /127.0.0.1:38878 [Receiving block BP-289274094-172.17.0.2-1731187411535:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:36507:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38878 dst: /127.0.0.1:36507
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:46,836 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1943374404_22 at /127.0.0.1:38864 [Receiving block BP-289274094-172.17.0.2-1731187411535:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:36507:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38864 dst: /127.0.0.1:36507
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:46,836 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1133754701_22 at /127.0.0.1:38892 [Receiving block BP-289274094-172.17.0.2-1731187411535:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:36507:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38892 dst: /127.0.0.1:36507
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:46,838 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@68c19d09{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:23:46,838 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@64434c96{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:23:46,838 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:23:46,838 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7bb583ee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:23:46,838 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@23d454c3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.log.dir/,STOPPED}
2024-11-09T21:23:46,839 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:23:46,839 WARN  [BP-289274094-172.17.0.2-1731187411535 heartbeating to localhost/127.0.0.1:38013 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:23:46,839 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:23:46,839 WARN  [BP-289274094-172.17.0.2-1731187411535 heartbeating to localhost/127.0.0.1:38013 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-289274094-172.17.0.2-1731187411535 (Datanode Uuid 5e4f120a-8b81-4cb2-88f9-73f064cc8cf1) service to localhost/127.0.0.1:38013
2024-11-09T21:23:46,840 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/data/data1/current/BP-289274094-172.17.0.2-1731187411535 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:23:46,840 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/data/data2/current/BP-289274094-172.17.0.2-1731187411535 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:23:46,840 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:23:46,848 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:23:46,851 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:23:46,851 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:23:46,851 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:23:46,852 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-09T21:23:46,852 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@701cf8bd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:23:46,852 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1fb64c0d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:23:46,942 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5cf288ef{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/java.io.tmpdir/jetty-localhost-42757-hadoop-hdfs-3_4_1-tests_jar-_-any-12265348068312380779/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:23:46,943 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@64e298ce{HTTP/1.1, (http/1.1)}{localhost:42757}
2024-11-09T21:23:46,943 INFO  [Time-limited test {}] server.Server(415): Started @174362ms
2024-11-09T21:23:46,944 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:23:47,205 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:47,269 WARN  [Thread-1326 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:23:47,272 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x77b27215dd688c4 with lease ID 0xb8b18762ab377c51: from storage DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2 node DatanodeRegistration(127.0.0.1:38031, datanodeUuid=e7072753-4f01-4a69-9c7c-d5479a6d7d5d, infoPort=42443, infoSecurePort=0, ipcPort=37371, storageInfo=lv=-57;cid=testClusterID;nsid=1886735975;c=1731187411535), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:23:47,272 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x77b27215dd688c4 with lease ID 0xb8b18762ab377c51: from storage DS-4240ffac-7920-4166-879f-9db6efc5937c node DatanodeRegistration(127.0.0.1:38031, datanodeUuid=e7072753-4f01-4a69-9c7c-d5479a6d7d5d, infoPort=42443, infoSecurePort=0, ipcPort=37371, storageInfo=lv=-57;cid=testClusterID;nsid=1886735975;c=1731187411535), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:23:47,343 WARN  [Thread-1346 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:23:47,345 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd109fe9a16e11c9b with lease ID 0xb8b18762ab377c52: from storage DS-8036443f-aa99-456b-bc73-67ff7f3fc77f node DatanodeRegistration(127.0.0.1:34083, datanodeUuid=5e4f120a-8b81-4cb2-88f9-73f064cc8cf1, infoPort=40147, infoSecurePort=0, ipcPort=36689, storageInfo=lv=-57;cid=testClusterID;nsid=1886735975;c=1731187411535), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:23:47,345 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd109fe9a16e11c9b with lease ID 0xb8b18762ab377c52: from storage DS-82ccbe06-7608-4295-b707-479f1f7198ce node DatanodeRegistration(127.0.0.1:34083, datanodeUuid=5e4f120a-8b81-4cb2-88f9-73f064cc8cf1, infoPort=40147, infoSecurePort=0, ipcPort=36689, storageInfo=lv=-57;cid=testClusterID;nsid=1886735975;c=1731187411535), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-09T21:23:47,464 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:47,486 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:47,966 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-11-09T21:23:47,970 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-11-09T21:23:47,973 ERROR [FSHLog-0-hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2-prefix:d86ff69d6719,45987,1731187413346 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36507,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:47,973 WARN  [FSHLog-0-hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2-prefix:d86ff69d6719,45987,1731187413346 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36507,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:47,973 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d86ff69d6719%2C45987%2C1731187413346:(num 1731187413863) roll requested
2024-11-09T21:23:47,974 INFO  [regionserver/d86ff69d6719:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C45987%2C1731187413346.1731187427973
2024-11-09T21:23:47,979 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187413863 newFile=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973
2024-11-09T21:23:47,979 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:47,979 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:47,980 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:47,980 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:47,980 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:47,980 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187413863 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973
2024-11-09T21:23:47,980 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36507,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:47,980 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36507,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:47,980 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187413863
2024-11-09T21:23:47,981 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40147:40147),(127.0.0.1/127.0.0.1:42443:42443)]
2024-11-09T21:23:47,981 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187413863 is not closed yet, will try archiving it next time
2024-11-09T21:23:47,981 WARN  [IPC Server handler 2 on default port 38013 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187413863 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1014
2024-11-09T21:23:47,981 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187413863 after 1ms
2024-11-09T21:23:48,206 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:48,465 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:48,487 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:49,207 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:49,466 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:49,488 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:49,987 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-11-09T21:23:50,209 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:50,274 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1014: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-09T21:23:50,467 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:50,489 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:51,210 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:51,468 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:51,490 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:51,983 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187413863 after 4003ms
2024-11-09T21:23:51,995 WARN  [ResponseProcessor for block BP-289274094-172.17.0.2-1731187411535:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-289274094-172.17.0.2-1731187411535:blk_1073741837_1016
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:51,996 WARN  [DataStreamer for file /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973 block BP-289274094-172.17.0.2-1731187411535:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-289274094-172.17.0.2-1731187411535:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34083,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK], DatanodeInfoWithStorage[127.0.0.1:38031,DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34083,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK]) is bad.
2024-11-09T21:23:51,997 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1943374404_22 at /127.0.0.1:33320 [Receiving block BP-289274094-172.17.0.2-1731187411535:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:34083:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33320 dst: /127.0.0.1:34083
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:51,997 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1943374404_22 at /127.0.0.1:48436 [Receiving block BP-289274094-172.17.0.2-1731187411535:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:38031:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48436 dst: /127.0.0.1:38031
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:52,015 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5cf288ef{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:23:52,016 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@64e298ce{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:23:52,016 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:23:52,016 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1fb64c0d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:23:52,017 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@701cf8bd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.log.dir/,STOPPED}
2024-11-09T21:23:52,019 WARN  [BP-289274094-172.17.0.2-1731187411535 heartbeating to localhost/127.0.0.1:38013 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:23:52,019 WARN  [BP-289274094-172.17.0.2-1731187411535 heartbeating to localhost/127.0.0.1:38013 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-289274094-172.17.0.2-1731187411535 (Datanode Uuid 5e4f120a-8b81-4cb2-88f9-73f064cc8cf1) service to localhost/127.0.0.1:38013
2024-11-09T21:23:52,019 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:23:52,019 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:23:52,021 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/data/data1/current/BP-289274094-172.17.0.2-1731187411535 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:23:52,021 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/data/data2/current/BP-289274094-172.17.0.2-1731187411535 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:23:52,021 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:23:52,031 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:23:52,035 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:23:52,036 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:23:52,036 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:23:52,036 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-09T21:23:52,037 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20da775a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:23:52,037 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e79e191{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:23:52,128 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@75d9c3d8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/java.io.tmpdir/jetty-localhost-34169-hadoop-hdfs-3_4_1-tests_jar-_-any-6548812171379913208/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:23:52,129 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@21ffcd24{HTTP/1.1, (http/1.1)}{localhost:34169}
2024-11-09T21:23:52,129 INFO  [Time-limited test {}] server.Server(415): Started @179548ms
2024-11-09T21:23:52,130 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:23:52,151 WARN  [ResponseProcessor for block BP-289274094-172.17.0.2-1731187411535:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-289274094-172.17.0.2-1731187411535:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:52,151 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1943374404_22 at /127.0.0.1:52762 [Receiving block BP-289274094-172.17.0.2-1731187411535:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:38031:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52762 dst: /127.0.0.1:38031
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:52,159 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5a362569{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:23:52,159 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@49f94f8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:23:52,160 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:23:52,160 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4022a798{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:23:52,160 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a40ff76{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.log.dir/,STOPPED}
2024-11-09T21:23:52,161 WARN  [BP-289274094-172.17.0.2-1731187411535 heartbeating to localhost/127.0.0.1:38013 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:23:52,161 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:23:52,161 WARN  [BP-289274094-172.17.0.2-1731187411535 heartbeating to localhost/127.0.0.1:38013 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-289274094-172.17.0.2-1731187411535 (Datanode Uuid e7072753-4f01-4a69-9c7c-d5479a6d7d5d) service to localhost/127.0.0.1:38013
2024-11-09T21:23:52,161 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:23:52,161 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/data/data3/current/BP-289274094-172.17.0.2-1731187411535 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:23:52,162 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/data/data4/current/BP-289274094-172.17.0.2-1731187411535 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:23:52,162 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:23:52,169 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:23:52,172 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:23:52,173 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:23:52,173 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:23:52,173 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T21:23:52,173 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3e7a090c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:23:52,174 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c90997a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:23:52,211 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:52,265 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1f27f35f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/java.io.tmpdir/jetty-localhost-33579-hadoop-hdfs-3_4_1-tests_jar-_-any-6167130059008815477/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:23:52,266 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22fd605f{HTTP/1.1, (http/1.1)}{localhost:33579}
2024-11-09T21:23:52,266 INFO  [Time-limited test {}] server.Server(415): Started @179684ms
2024-11-09T21:23:52,267 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:23:52,469 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:52,491 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:52,517 WARN  [Thread-1400 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:23:52,519 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x25c3a320a89b663d with lease ID 0xb8b18762ab377c53: from storage DS-8036443f-aa99-456b-bc73-67ff7f3fc77f node DatanodeRegistration(127.0.0.1:43181, datanodeUuid=5e4f120a-8b81-4cb2-88f9-73f064cc8cf1, infoPort=33139, infoSecurePort=0, ipcPort=34427, storageInfo=lv=-57;cid=testClusterID;nsid=1886735975;c=1731187411535), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:23:52,519 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x25c3a320a89b663d with lease ID 0xb8b18762ab377c53: from storage DS-82ccbe06-7608-4295-b707-479f1f7198ce node DatanodeRegistration(127.0.0.1:43181, datanodeUuid=5e4f120a-8b81-4cb2-88f9-73f064cc8cf1, infoPort=33139, infoSecurePort=0, ipcPort=34427, storageInfo=lv=-57;cid=testClusterID;nsid=1886735975;c=1731187411535), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-09T21:23:52,647 WARN  [Thread-1420 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:23:52,649 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xed9fe7f7107fcd7b with lease ID 0xb8b18762ab377c54: from storage DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2 node DatanodeRegistration(127.0.0.1:32921, datanodeUuid=e7072753-4f01-4a69-9c7c-d5479a6d7d5d, infoPort=38173, infoSecurePort=0, ipcPort=37923, storageInfo=lv=-57;cid=testClusterID;nsid=1886735975;c=1731187411535), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:23:52,649 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xed9fe7f7107fcd7b with lease ID 0xb8b18762ab377c54: from storage DS-4240ffac-7920-4166-879f-9db6efc5937c node DatanodeRegistration(127.0.0.1:32921, datanodeUuid=e7072753-4f01-4a69-9c7c-d5479a6d7d5d, infoPort=38173, infoSecurePort=0, ipcPort=37923, storageInfo=lv=-57;cid=testClusterID;nsid=1886735975;c=1731187411535), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-09T21:23:53,212 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:53,282 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-11-09T21:23:53,284 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-11-09T21:23:53,285 ERROR [FSHLog-0-hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2-prefix:d86ff69d6719,45987,1731187413346 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38031,DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:53,286 WARN  [FSHLog-0-hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2-prefix:d86ff69d6719,45987,1731187413346 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38031,DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:53,286 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d86ff69d6719%2C45987%2C1731187413346:(num 1731187427973) roll requested
2024-11-09T21:23:53,286 INFO  [regionserver/d86ff69d6719:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C45987%2C1731187413346.1731187433286
2024-11-09T21:23:53,294 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973 newFile=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187433286
2024-11-09T21:23:53,294 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:53,295 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:53,295 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:53,295 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:53,295 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:53,295 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187433286
2024-11-09T21:23:53,295 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38031,DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:53,295 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38031,DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:53,296 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973
2024-11-09T21:23:53,296 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38173:38173),(127.0.0.1/127.0.0.1:33139:33139)]
2024-11-09T21:23:53,296 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973 is not closed yet, will try archiving it next time
2024-11-09T21:23:53,296 WARN  [IPC Server handler 0 on default port 38013 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-11-09T21:23:53,296 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973 after 0ms
2024-11-09T21:23:53,470 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:53,492 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:54,213 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:54,472 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:54,493 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:55,215 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:55,298 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C45987%2C1731187413346.1731187435298
2024-11-09T21:23:55,311 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187433286 newFile=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298
2024-11-09T21:23:55,311 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:55,311 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:55,312 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:55,312 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:55,312 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:55,312 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187433286 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298
2024-11-09T21:23:55,313 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38173:38173),(127.0.0.1/127.0.0.1:33139:33139)]
2024-11-09T21:23:55,313 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973 is not closed yet, will try archiving it next time
2024-11-09T21:23:55,313 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187433286 is not closed yet, will try archiving it next time
2024-11-09T21:23:55,313 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187413863
2024-11-09T21:23:55,314 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187413863
2024-11-09T21:23:55,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43181 is added to blk_1073741838_1019 (size=1264)
2024-11-09T21:23:55,314 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187413863 after 0ms
2024-11-09T21:23:55,314 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187413863
2024-11-09T21:23:55,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32921 is added to blk_1073741838_1019 (size=1264)
2024-11-09T21:23:55,315 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973 is not closed yet, will try archiving it next time
2024-11-09T21:23:55,323 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1731187414860/Put/vlen=218/seqid=0]
2024-11-09T21:23:55,323 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1731187424591/Put/vlen=1045/seqid=0]
2024-11-09T21:23:55,323 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187413863
2024-11-09T21:23:55,323 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973
2024-11-09T21:23:55,323 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973
2024-11-09T21:23:55,324 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973 after 1ms
2024-11-09T21:23:55,324 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973
2024-11-09T21:23:55,328 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1731187427972/Put/vlen=1045/seqid=0]
2024-11-09T21:23:55,328 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1731187429990/Put/vlen=1045/seqid=0]
2024-11-09T21:23:55,328 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973
2024-11-09T21:23:55,328 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187433286
2024-11-09T21:23:55,328 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187433286
2024-11-09T21:23:55,328 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187433286 after 0ms
2024-11-09T21:23:55,328 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187433286
2024-11-09T21:23:55,331 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1731187433285/Put/vlen=1045/seqid=0]
2024-11-09T21:23:55,331 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298
2024-11-09T21:23:55,331 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298
2024-11-09T21:23:55,332 WARN  [IPC Server handler 4 on default port 38013 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-11-09T21:23:55,332 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298 after 1ms
2024-11-09T21:23:55,473 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:55,494 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:55,653 WARN  [ResponseProcessor for block BP-289274094-172.17.0.2-1731187411535:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-289274094-172.17.0.2-1731187411535:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:55,653 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1133754701_22 at /127.0.0.1:60094 [Receiving block BP-289274094-172.17.0.2-1731187411535:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:32921:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60094 dst: /127.0.0.1:32921
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:32921 remote=/127.0.0.1:60094]. Total timeout mills is 60000, 59658 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:55,654 WARN  [DataStreamer for file /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298 block BP-289274094-172.17.0.2-1731187411535:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-289274094-172.17.0.2-1731187411535:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32921,DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2,DISK], DatanodeInfoWithStorage[127.0.0.1:43181,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32921,DS-facc54b9-5095-4adb-ab8f-370ab50a4cf2,DISK]) is bad.
2024-11-09T21:23:55,654 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1133754701_22 at /127.0.0.1:53322 [Receiving block BP-289274094-172.17.0.2-1731187411535:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:43181:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53322 dst: /127.0.0.1:43181
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:23:55,657 WARN  [DataStreamer for file /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298 block BP-289274094-172.17.0.2-1731187411535:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-289274094-172.17.0.2-1731187411535:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:55,664 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32921 is added to blk_1073741839_1022 (size=85)
2024-11-09T21:23:55,666 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43181 is added to blk_1073741839_1022 (size=85)
2024-11-09T21:23:56,216 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:56,474 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:56,495 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:56,521 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-09T21:23:57,217 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:57,299 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187427973 after 4003ms
2024-11-09T21:23:57,476 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:57,496 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:58,219 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:58,477 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:58,497 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:59,220 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:59,334 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298 after 4003ms
2024-11-09T21:23:59,334 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298
2024-11-09T21:23:59,343 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298
2024-11-09T21:23:59,343 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 595d6bfca74458389c5bf371f7c86a09 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-09T21:23:59,343 ERROR [FSHLog-0-hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2-prefix:d86ff69d6719,45987,1731187413346 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-289274094-172.17.0.2-1731187411535:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:59,344 WARN  [FSHLog-0-hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2-prefix:d86ff69d6719,45987,1731187413346 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-289274094-172.17.0.2-1731187411535:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:59,344 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d86ff69d6719%2C45987%2C1731187413346:(num 1731187435298) roll requested
2024-11-09T21:23:59,344 INFO  [regionserver/d86ff69d6719:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C45987%2C1731187413346.1731187439344
2024-11-09T21:23:59,349 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298 newFile=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187439344
2024-11-09T21:23:59,350 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:59,350 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:59,350 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:59,350 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:59,350 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:59,350 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187439344
2024-11-09T21:23:59,350 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-289274094-172.17.0.2-1731187411535:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:59,351 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33139:33139),(127.0.0.1/127.0.0.1:38173:38173)]
2024-11-09T21:23:59,351 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298 is not closed yet, will try archiving it next time
2024-11-09T21:23:59,351 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-289274094-172.17.0.2-1731187411535:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:59,351 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298
2024-11-09T21:23:59,352 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298 after 1ms
2024-11-09T21:23:59,352 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.1731187435298 to hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/oldWALs/d86ff69d6719%2C45987%2C1731187413346.1731187435298
2024-11-09T21:23:59,371 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/default/TestLogRolling-testLogRollOnPipelineRestart/595d6bfca74458389c5bf371f7c86a09/.tmp/info/9872671c500d470fb37e951907468bb6 is 1080, key is row1002/info:/1731187424591/Put/seqid=0
2024-11-09T21:23:59,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43181 is added to blk_1073741841_1024 (size=9270)
2024-11-09T21:23:59,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32921 is added to blk_1073741841_1024 (size=9270)
2024-11-09T21:23:59,377 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/default/TestLogRolling-testLogRollOnPipelineRestart/595d6bfca74458389c5bf371f7c86a09/.tmp/info/9872671c500d470fb37e951907468bb6
2024-11-09T21:23:59,384 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/default/TestLogRolling-testLogRollOnPipelineRestart/595d6bfca74458389c5bf371f7c86a09/.tmp/info/9872671c500d470fb37e951907468bb6 as hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/default/TestLogRolling-testLogRollOnPipelineRestart/595d6bfca74458389c5bf371f7c86a09/info/9872671c500d470fb37e951907468bb6
2024-11-09T21:23:59,390 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/default/TestLogRolling-testLogRollOnPipelineRestart/595d6bfca74458389c5bf371f7c86a09/info/9872671c500d470fb37e951907468bb6, entries=4, sequenceid=8, filesize=9.1 K
2024-11-09T21:23:59,391 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 595d6bfca74458389c5bf371f7c86a09 in 48ms, sequenceid=8, compaction requested=false
2024-11-09T21:23:59,391 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 595d6bfca74458389c5bf371f7c86a09:

2024-11-09T21:23:59,391 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-11-09T21:23:59,392 ERROR [FSHLog-0-hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2-prefix:d86ff69d6719,45987,1731187413346.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36507,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:59,392 WARN  [FSHLog-0-hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2-prefix:d86ff69d6719,45987,1731187413346.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36507,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:59,392 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d86ff69d6719%2C45987%2C1731187413346.meta:.meta(num 1731187414320) roll requested
2024-11-09T21:23:59,392 INFO  [regionserver/d86ff69d6719:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C45987%2C1731187413346.meta.1731187439392.meta
2024-11-09T21:23:59,397 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:59,397 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:59,397 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:59,398 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:59,398 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:23:59,398 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.meta.1731187414320.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.meta.1731187439392.meta
2024-11-09T21:23:59,398 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36507,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:59,398 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36507,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:23:59,398 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.meta.1731187414320.meta
2024-11-09T21:23:59,399 WARN  [IPC Server handler 1 on default port 38013 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.meta.1731187414320.meta has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741834_1013
2024-11-09T21:23:59,399 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.meta.1731187414320.meta after 1ms
2024-11-09T21:23:59,402 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38173:38173),(127.0.0.1/127.0.0.1:33139:33139)]
2024-11-09T21:23:59,402 DEBUG [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.meta.1731187414320.meta is not closed yet, will try archiving it next time
2024-11-09T21:23:59,422 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/.tmp/info/ca4cd2e15172420cbc5cd5e2563ad0cf is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09./info:regioninfo/1731187414865/Put/seqid=0
2024-11-09T21:23:59,427 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32921 is added to blk_1073741843_1027 (size=7125)
2024-11-09T21:23:59,427 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43181 is added to blk_1073741843_1027 (size=7125)
2024-11-09T21:23:59,428 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/.tmp/info/ca4cd2e15172420cbc5cd5e2563ad0cf
2024-11-09T21:23:59,448 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/.tmp/ns/584f73fdc79e4c96a5c3a9ba37fea294 is 43, key is default/ns:d/1731187414398/Put/seqid=0
2024-11-09T21:23:59,452 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43181 is added to blk_1073741844_1028 (size=5153)
2024-11-09T21:23:59,452 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32921 is added to blk_1073741844_1028 (size=5153)
2024-11-09T21:23:59,453 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/.tmp/ns/584f73fdc79e4c96a5c3a9ba37fea294
2024-11-09T21:23:59,472 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/.tmp/table/3c686ee23f0d450389b3017e544c1b70 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1731187414875/Put/seqid=0
2024-11-09T21:23:59,477 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32921 is added to blk_1073741845_1029 (size=5438)
2024-11-09T21:23:59,477 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43181 is added to blk_1073741845_1029 (size=5438)
2024-11-09T21:23:59,477 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:59,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:23:59,879 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/.tmp/table/3c686ee23f0d450389b3017e544c1b70
2024-11-09T21:23:59,892 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/.tmp/info/ca4cd2e15172420cbc5cd5e2563ad0cf as hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/info/ca4cd2e15172420cbc5cd5e2563ad0cf
2024-11-09T21:23:59,899 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/info/ca4cd2e15172420cbc5cd5e2563ad0cf, entries=10, sequenceid=11, filesize=7.0 K
2024-11-09T21:23:59,900 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/.tmp/ns/584f73fdc79e4c96a5c3a9ba37fea294 as hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/ns/584f73fdc79e4c96a5c3a9ba37fea294
2024-11-09T21:23:59,906 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/ns/584f73fdc79e4c96a5c3a9ba37fea294, entries=2, sequenceid=11, filesize=5.0 K
2024-11-09T21:23:59,908 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/.tmp/table/3c686ee23f0d450389b3017e544c1b70 as hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/table/3c686ee23f0d450389b3017e544c1b70
2024-11-09T21:23:59,914 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/table/3c686ee23f0d450389b3017e544c1b70, entries=2, sequenceid=11, filesize=5.3 K
2024-11-09T21:23:59,915 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 524ms, sequenceid=11, compaction requested=false
2024-11-09T21:23:59,916 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-09T21:23:59,921 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-09T21:23:59,921 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-09T21:23:59,921 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:23:59,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:23:59,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:23:59,922 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-09T21:23:59,922 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-09T21:23:59,922 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1808109364, stopped=false
2024-11-09T21:23:59,922 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d86ff69d6719,35765,1731187413188
2024-11-09T21:23:59,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-09T21:23:59,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-09T21:23:59,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:59,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:23:59,973 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-09T21:23:59,973 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-09T21:23:59,974 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:23:59,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:23:59,974 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd86ff69d6719,45987,1731187413346' *****
2024-11-09T21:23:59,974 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-09T21:23:59,975 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:23:59,975 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:23:59,975 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-09T21:23:59,975 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-09T21:23:59,976 INFO  [RS:0;d86ff69d6719:45987 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-09T21:23:59,976 INFO  [RS:0;d86ff69d6719:45987 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-09T21:23:59,976 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer(3091): Received CLOSE for 595d6bfca74458389c5bf371f7c86a09
2024-11-09T21:23:59,977 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer(959): stopping server d86ff69d6719,45987,1731187413346
2024-11-09T21:23:59,977 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-09T21:23:59,977 INFO  [RS:0;d86ff69d6719:45987 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d86ff69d6719:45987.
2024-11-09T21:23:59,977 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 595d6bfca74458389c5bf371f7c86a09, disabling compactions & flushes
2024-11-09T21:23:59,977 DEBUG [RS:0;d86ff69d6719:45987 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:23:59,977 DEBUG [RS:0;d86ff69d6719:45987 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:23:59,977 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.
2024-11-09T21:23:59,977 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.
2024-11-09T21:23:59,977 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09. after waiting 0 ms
2024-11-09T21:23:59,977 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-09T21:23:59,977 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.
2024-11-09T21:23:59,977 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-09T21:23:59,977 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-09T21:23:59,977 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-09T21:23:59,978 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-09T21:23:59,978 DEBUG [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer(1325): Online Regions={595d6bfca74458389c5bf371f7c86a09=TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09., 1588230740=hbase:meta,,1.1588230740}
2024-11-09T21:23:59,978 DEBUG [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 595d6bfca74458389c5bf371f7c86a09
2024-11-09T21:23:59,978 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-09T21:23:59,978 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-09T21:23:59,978 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-09T21:23:59,978 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-09T21:23:59,978 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-09T21:23:59,984 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/default/TestLogRolling-testLogRollOnPipelineRestart/595d6bfca74458389c5bf371f7c86a09/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-11-09T21:23:59,984 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-09T21:23:59,984 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.
2024-11-09T21:23:59,985 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 595d6bfca74458389c5bf371f7c86a09:
Waiting for close lock at 1731187439977Running coprocessor pre-close hooks at 1731187439977Disabling compacts and flushes for region at 1731187439977Disabling writes for close at 1731187439977Writing region close event to WAL at 1731187439978 (+1 ms)Running coprocessor post-close hooks at 1731187439984 (+6 ms)Closed at 1731187439984
2024-11-09T21:23:59,985 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731187414492.595d6bfca74458389c5bf371f7c86a09.
2024-11-09T21:23:59,985 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-09T21:23:59,985 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-09T21:23:59,985 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731187439978Running coprocessor pre-close hooks at 1731187439978Disabling compacts and flushes for region at 1731187439978Disabling writes for close at 1731187439978Writing region close event to WAL at 1731187439980 (+2 ms)Running coprocessor post-close hooks at 1731187439985 (+5 ms)Closed at 1731187439985
2024-11-09T21:23:59,985 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-09T21:24:00,178 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer(976): stopping server d86ff69d6719,45987,1731187413346; all regions closed.
2024-11-09T21:24:00,179 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:00,180 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:00,180 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:00,180 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:00,181 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:00,185 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43181 is added to blk_1073741842_1025 (size=825)
2024-11-09T21:24:00,186 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32921 is added to blk_1073741842_1025 (size=825)
2024-11-09T21:24:00,221 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:00,478 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:00,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:00,772 INFO  [regionserver/d86ff69d6719:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-09T21:24:00,772 INFO  [regionserver/d86ff69d6719:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-09T21:24:01,222 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:01,480 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:01,499 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:01,733 INFO  [regionserver/d86ff69d6719:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-09T21:24:02,223 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:02,225 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880 after 68091ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor191.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:24:02,481 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:02,501 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:02,654 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1013: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-09T21:24:03,170 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T21:24:03,226 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:03,401 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.meta.1731187414320.meta after 4003ms
2024-11-09T21:24:03,402 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/WALs/d86ff69d6719,45987,1731187413346/d86ff69d6719%2C45987%2C1731187413346.meta.1731187414320.meta to hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/oldWALs/d86ff69d6719%2C45987%2C1731187413346.meta.1731187414320.meta
2024-11-09T21:24:03,411 DEBUG [RS:0;d86ff69d6719:45987 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/oldWALs
2024-11-09T21:24:03,411 INFO  [RS:0;d86ff69d6719:45987 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d86ff69d6719%2C45987%2C1731187413346.meta:.meta(num 1731187439392)
2024-11-09T21:24:03,412 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:03,412 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:03,412 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:03,413 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:03,413 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:03,415 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32921 is added to blk_1073741840_1023 (size=1162)
2024-11-09T21:24:03,416 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43181 is added to blk_1073741840_1023 (size=1162)
2024-11-09T21:24:03,421 DEBUG [RS:0;d86ff69d6719:45987 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/oldWALs
2024-11-09T21:24:03,421 INFO  [RS:0;d86ff69d6719:45987 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d86ff69d6719%2C45987%2C1731187413346:(num 1731187439344)
2024-11-09T21:24:03,421 DEBUG [RS:0;d86ff69d6719:45987 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:24:03,421 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.LeaseManager(133): Closed leases
2024-11-09T21:24:03,421 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-09T21:24:03,421 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.ChoreService(370): Chore service for: regionserver/d86ff69d6719:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-09T21:24:03,421 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-09T21:24:03,421 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-09T21:24:03,421 INFO  [RS:0;d86ff69d6719:45987 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45987
2024-11-09T21:24:03,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T21:24:03,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d86ff69d6719,45987,1731187413346
2024-11-09T21:24:03,471 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-09T21:24:03,480 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d86ff69d6719,45987,1731187413346]
2024-11-09T21:24:03,482 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:03,488 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d86ff69d6719,45987,1731187413346 already deleted, retry=false
2024-11-09T21:24:03,488 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d86ff69d6719,45987,1731187413346 expired; onlineServers=0
2024-11-09T21:24:03,488 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd86ff69d6719,35765,1731187413188' *****
2024-11-09T21:24:03,488 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-09T21:24:03,488 INFO  [M:0;d86ff69d6719:35765 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-09T21:24:03,488 INFO  [M:0;d86ff69d6719:35765 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-09T21:24:03,488 DEBUG [M:0;d86ff69d6719:35765 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-09T21:24:03,488 DEBUG [M:0;d86ff69d6719:35765 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-09T21:24:03,488 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-09T21:24:03,488 DEBUG [master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187413654 {}] cleaner.HFileCleaner(306): Exit Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187413654,5,FailOnTimeoutGroup]
2024-11-09T21:24:03,489 DEBUG [master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187413654 {}] cleaner.HFileCleaner(306): Exit Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187413654,5,FailOnTimeoutGroup]
2024-11-09T21:24:03,489 INFO  [M:0;d86ff69d6719:35765 {}] hbase.ChoreService(370): Chore service for: master/d86ff69d6719:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-09T21:24:03,489 INFO  [M:0;d86ff69d6719:35765 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-09T21:24:03,489 DEBUG [M:0;d86ff69d6719:35765 {}] master.HMaster(1795): Stopping service threads
2024-11-09T21:24:03,489 INFO  [M:0;d86ff69d6719:35765 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-09T21:24:03,489 INFO  [M:0;d86ff69d6719:35765 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-09T21:24:03,490 INFO  [M:0;d86ff69d6719:35765 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-09T21:24:03,490 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-09T21:24:03,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-09T21:24:03,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:24:03,497 DEBUG [M:0;d86ff69d6719:35765 {}] zookeeper.ZKUtil(347): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-09T21:24:03,497 WARN  [M:0;d86ff69d6719:35765 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-09T21:24:03,498 INFO  [M:0;d86ff69d6719:35765 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/.lastflushedseqids
2024-11-09T21:24:03,501 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:03,506 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32921 is added to blk_1073741846_1030 (size=111)
2024-11-09T21:24:03,507 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43181 is added to blk_1073741846_1030 (size=111)
2024-11-09T21:24:03,507 INFO  [M:0;d86ff69d6719:35765 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-09T21:24:03,507 INFO  [M:0;d86ff69d6719:35765 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-09T21:24:03,508 DEBUG [M:0;d86ff69d6719:35765 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-09T21:24:03,508 INFO  [M:0;d86ff69d6719:35765 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:24:03,508 DEBUG [M:0;d86ff69d6719:35765 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:24:03,508 DEBUG [M:0;d86ff69d6719:35765 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-09T21:24:03,508 DEBUG [M:0;d86ff69d6719:35765 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:24:03,508 INFO  [M:0;d86ff69d6719:35765 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.18 KB heapSize=29.16 KB
2024-11-09T21:24:03,509 ERROR [FSHLog-0-hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData-prefix:d86ff69d6719,35765,1731187413188 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36507,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:24:03,509 WARN  [FSHLog-0-hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData-prefix:d86ff69d6719,35765,1731187413188 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36507,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:24:03,509 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog d86ff69d6719%2C35765%2C1731187413188:(num 1731187413472) roll requested
2024-11-09T21:24:03,509 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C35765%2C1731187413188.1731187443509
2024-11-09T21:24:03,516 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:03,517 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:03,517 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:03,517 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:03,517 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:03,517 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/WALs/d86ff69d6719,35765,1731187413188/d86ff69d6719%2C35765%2C1731187413188.1731187413472 with entries=53, filesize=26.63 KB; new WAL /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/WALs/d86ff69d6719,35765,1731187413188/d86ff69d6719%2C35765%2C1731187413188.1731187443509
2024-11-09T21:24:03,517 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36507,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:24:03,518 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36507,DS-8036443f-aa99-456b-bc73-67ff7f3fc77f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-09T21:24:03,518 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/WALs/d86ff69d6719,35765,1731187413188/d86ff69d6719%2C35765%2C1731187413188.1731187413472
2024-11-09T21:24:03,518 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33139:33139),(127.0.0.1/127.0.0.1:38173:38173)]
2024-11-09T21:24:03,518 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/WALs/d86ff69d6719,35765,1731187413188/d86ff69d6719%2C35765%2C1731187413188.1731187413472 is not closed yet, will try archiving it next time
2024-11-09T21:24:03,518 WARN  [IPC Server handler 1 on default port 38013 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/WALs/d86ff69d6719,35765,1731187413188/d86ff69d6719%2C35765%2C1731187413188.1731187413472 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015
2024-11-09T21:24:03,518 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/WALs/d86ff69d6719,35765,1731187413188/d86ff69d6719%2C35765%2C1731187413188.1731187413472 after 0ms
2024-11-09T21:24:03,520 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43181 is added to blk_1073741830_1032 (size=27271)
2024-11-09T21:24:03,532 DEBUG [M:0;d86ff69d6719:35765 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/83f0282e612f434dac02b8244c3a18bd is 82, key is hbase:meta,,1/info:regioninfo/1731187414346/Put/seqid=0
2024-11-09T21:24:03,537 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32921 is added to blk_1073741848_1033 (size=5672)
2024-11-09T21:24:03,537 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43181 is added to blk_1073741848_1033 (size=5672)
2024-11-09T21:24:03,537 INFO  [M:0;d86ff69d6719:35765 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/83f0282e612f434dac02b8244c3a18bd
2024-11-09T21:24:03,555 DEBUG [M:0;d86ff69d6719:35765 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c5046b1a71f94385ab0fe4fbb47e51db is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731187414879/Put/seqid=0
2024-11-09T21:24:03,559 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32921 is added to blk_1073741849_1034 (size=6119)
2024-11-09T21:24:03,559 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43181 is added to blk_1073741849_1034 (size=6119)
2024-11-09T21:24:03,560 INFO  [M:0;d86ff69d6719:35765 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c5046b1a71f94385ab0fe4fbb47e51db
2024-11-09T21:24:03,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:24:03,580 INFO  [RS:0;d86ff69d6719:45987 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-09T21:24:03,580 DEBUG [M:0;d86ff69d6719:35765 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ec53b716ab024957b7f113a9156baaf6 is 69, key is d86ff69d6719,45987,1731187413346/rs:state/1731187413711/Put/seqid=0
2024-11-09T21:24:03,580 INFO  [RS:0;d86ff69d6719:45987 {}] regionserver.HRegionServer(1031): Exiting; stopping=d86ff69d6719,45987,1731187413346; zookeeper connection closed.
2024-11-09T21:24:03,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45987-0x10122c5237a0001, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:24:03,580 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@8474475 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@8474475
2024-11-09T21:24:03,580 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-09T21:24:03,584 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43181 is added to blk_1073741850_1035 (size=5156)
2024-11-09T21:24:03,584 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32921 is added to blk_1073741850_1035 (size=5156)
2024-11-09T21:24:03,585 INFO  [M:0;d86ff69d6719:35765 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ec53b716ab024957b7f113a9156baaf6
2024-11-09T21:24:03,603 DEBUG [M:0;d86ff69d6719:35765 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5e4e11a9f2c34527a7ce0f97b6927592 is 52, key is load_balancer_on/state:d/1731187414487/Put/seqid=0
2024-11-09T21:24:03,607 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43181 is added to blk_1073741851_1036 (size=5056)
2024-11-09T21:24:03,607 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32921 is added to blk_1073741851_1036 (size=5056)
2024-11-09T21:24:03,608 INFO  [M:0;d86ff69d6719:35765 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5e4e11a9f2c34527a7ce0f97b6927592
2024-11-09T21:24:03,613 DEBUG [M:0;d86ff69d6719:35765 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/83f0282e612f434dac02b8244c3a18bd as hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/83f0282e612f434dac02b8244c3a18bd
2024-11-09T21:24:03,618 INFO  [M:0;d86ff69d6719:35765 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/83f0282e612f434dac02b8244c3a18bd, entries=8, sequenceid=56, filesize=5.5 K
2024-11-09T21:24:03,619 DEBUG [M:0;d86ff69d6719:35765 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c5046b1a71f94385ab0fe4fbb47e51db as hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c5046b1a71f94385ab0fe4fbb47e51db
2024-11-09T21:24:03,624 INFO  [M:0;d86ff69d6719:35765 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c5046b1a71f94385ab0fe4fbb47e51db, entries=6, sequenceid=56, filesize=6.0 K
2024-11-09T21:24:03,626 DEBUG [M:0;d86ff69d6719:35765 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ec53b716ab024957b7f113a9156baaf6 as hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ec53b716ab024957b7f113a9156baaf6
2024-11-09T21:24:03,630 INFO  [M:0;d86ff69d6719:35765 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ec53b716ab024957b7f113a9156baaf6, entries=1, sequenceid=56, filesize=5.0 K
2024-11-09T21:24:03,632 DEBUG [M:0;d86ff69d6719:35765 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5e4e11a9f2c34527a7ce0f97b6927592 as hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5e4e11a9f2c34527a7ce0f97b6927592
2024-11-09T21:24:03,636 INFO  [M:0;d86ff69d6719:35765 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5e4e11a9f2c34527a7ce0f97b6927592, entries=1, sequenceid=56, filesize=4.9 K
2024-11-09T21:24:03,637 INFO  [M:0;d86ff69d6719:35765 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.18 KB/23738, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 129ms, sequenceid=56, compaction requested=false
2024-11-09T21:24:03,639 INFO  [M:0;d86ff69d6719:35765 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:24:03,639 DEBUG [M:0;d86ff69d6719:35765 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731187443508Disabling compacts and flushes for region at 1731187443508Disabling writes for close at 1731187443508Obtaining lock to block concurrent updates at 1731187443508Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731187443508Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23738, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1731187443509 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731187443518 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731187443518Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731187443531 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731187443532 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731187443542 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731187443555 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731187443555Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731187443565 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731187443579 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731187443579Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731187443589 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731187443602 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731187443602Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d1153de: reopening flushed file at 1731187443612 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e3815bb: reopening flushed file at 1731187443618 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@791a7b07: reopening flushed file at 1731187443625 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ec30798: reopening flushed file at 1731187443631 (+6 ms)Finished flush of dataSize ~23.18 KB/23738, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 129ms, sequenceid=56, compaction requested=false at 1731187443637 (+6 ms)Writing region close event to WAL at 1731187443639 (+2 ms)Closed at 1731187443639
2024-11-09T21:24:03,639 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:03,639 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:03,639 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:03,639 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:03,639 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:03,641 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32921 is added to blk_1073741847_1031 (size=757)
2024-11-09T21:24:03,641 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43181 is added to blk_1073741847_1031 (size=757)
2024-11-09T21:24:04,227 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:04,482 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:04,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:04,986 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:04,986 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,007 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,007 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,007 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,007 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,008 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,008 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,011 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,011 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,011 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,012 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,016 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,016 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,228 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:05,483 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:05,503 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:05,520 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-09T21:24:05,522 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,523 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,524 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,524 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,543 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,543 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,543 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,544 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,544 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,544 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,547 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,547 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,547 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,549 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:05,654 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-09T21:24:06,229 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:06,485 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:06,504 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:07,231 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:07,486 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:07,505 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:07,520 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/WALs/d86ff69d6719,35765,1731187413188/d86ff69d6719%2C35765%2C1731187413188.1731187413472 after 4002ms
2024-11-09T21:24:07,521 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/WALs/d86ff69d6719,35765,1731187413188/d86ff69d6719%2C35765%2C1731187413188.1731187413472 to hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/oldWALs/d86ff69d6719%2C35765%2C1731187413188.1731187413472
2024-11-09T21:24:07,530 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/MasterData/oldWALs/d86ff69d6719%2C35765%2C1731187413188.1731187413472 to hdfs://localhost:38013/user/jenkins/test-data/28d3892f-c12f-a68c-9ff8-9c0f92793ae2/oldWALs/d86ff69d6719%2C35765%2C1731187413188.1731187413472$masterlocalwal$
2024-11-09T21:24:07,530 INFO  [M:0;d86ff69d6719:35765 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-09T21:24:07,530 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-09T21:24:07,530 INFO  [M:0;d86ff69d6719:35765 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35765
2024-11-09T21:24:07,530 INFO  [M:0;d86ff69d6719:35765 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-09T21:24:07,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:24:07,662 INFO  [M:0;d86ff69d6719:35765 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-09T21:24:07,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35765-0x10122c5237a0000, quorum=127.0.0.1:62742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:24:07,665 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1f27f35f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:24:07,665 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22fd605f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:24:07,666 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:24:07,666 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c90997a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:24:07,666 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3e7a090c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.log.dir/,STOPPED}
2024-11-09T21:24:07,667 WARN  [BP-289274094-172.17.0.2-1731187411535 heartbeating to localhost/127.0.0.1:38013 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:24:07,667 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:24:07,667 WARN  [BP-289274094-172.17.0.2-1731187411535 heartbeating to localhost/127.0.0.1:38013 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-289274094-172.17.0.2-1731187411535 (Datanode Uuid e7072753-4f01-4a69-9c7c-d5479a6d7d5d) service to localhost/127.0.0.1:38013
2024-11-09T21:24:07,667 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:24:07,668 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/data/data3/current/BP-289274094-172.17.0.2-1731187411535 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:24:07,668 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/data/data4/current/BP-289274094-172.17.0.2-1731187411535 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:24:07,668 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:24:07,671 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@75d9c3d8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:24:07,671 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@21ffcd24{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:24:07,671 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:24:07,671 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5e79e191{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:24:07,671 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20da775a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.log.dir/,STOPPED}
2024-11-09T21:24:07,673 WARN  [BP-289274094-172.17.0.2-1731187411535 heartbeating to localhost/127.0.0.1:38013 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:24:07,673 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:24:07,673 WARN  [BP-289274094-172.17.0.2-1731187411535 heartbeating to localhost/127.0.0.1:38013 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-289274094-172.17.0.2-1731187411535 (Datanode Uuid 5e4f120a-8b81-4cb2-88f9-73f064cc8cf1) service to localhost/127.0.0.1:38013
2024-11-09T21:24:07,673 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:24:07,673 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/data/data1/current/BP-289274094-172.17.0.2-1731187411535 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:24:07,673 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/cluster_089de005-d356-3a4a-dc4c-23d1f6fbc94b/data/data2/current/BP-289274094-172.17.0.2-1731187411535 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:24:07,674 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:24:07,679 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34c1099f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-09T21:24:07,679 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@56e801fe{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:24:07,679 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:24:07,679 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a6464d2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:24:07,680 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a011456{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.log.dir/,STOPPED}
2024-11-09T21:24:07,685 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-09T21:24:07,704 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-09T21:24:07,710 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=183 (was 158)
Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38013
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38013 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:38013
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:38013
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38013
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38013
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38013 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38013 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 436) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=44 (was 53), ProcessCount=11 (was 11), AvailableMemoryMB=18062 (was 18228)
2024-11-09T21:24:07,717 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=183, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=44, ProcessCount=11, AvailableMemoryMB=18062
2024-11-09T21:24:07,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-09T21:24:07,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.log.dir so I do NOT create it in target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928
2024-11-09T21:24:07,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e6288a6f-e12c-19db-b7e8-d4f6b2569342/hadoop.tmp.dir so I do NOT create it in target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928
2024-11-09T21:24:07,717 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/cluster_d0e49250-96c8-093e-ed32-ea6c84048991, deleteOnExit=true
2024-11-09T21:24:07,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-09T21:24:07,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/test.cache.data in system properties and HBase conf
2024-11-09T21:24:07,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/hadoop.tmp.dir in system properties and HBase conf
2024-11-09T21:24:07,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/hadoop.log.dir in system properties and HBase conf
2024-11-09T21:24:07,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-09T21:24:07,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-09T21:24:07,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-09T21:24:07,718 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-09T21:24:07,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-09T21:24:07,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-09T21:24:07,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-09T21:24:07,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-09T21:24:07,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-09T21:24:07,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-09T21:24:07,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-09T21:24:07,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-09T21:24:07,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-09T21:24:07,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/nfs.dump.dir in system properties and HBase conf
2024-11-09T21:24:07,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/java.io.tmpdir in system properties and HBase conf
2024-11-09T21:24:07,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-09T21:24:07,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-09T21:24:07,719 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-09T21:24:07,731 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-09T21:24:08,000 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:24:08,004 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:24:08,005 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:24:08,005 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:24:08,005 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T21:24:08,005 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:24:08,006 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8afa355{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:24:08,006 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6bd1231f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:24:08,096 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4e873b68{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/java.io.tmpdir/jetty-localhost-42657-hadoop-hdfs-3_4_1-tests_jar-_-any-11787008403859523958/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-09T21:24:08,096 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22fd1e24{HTTP/1.1, (http/1.1)}{localhost:42657}
2024-11-09T21:24:08,097 INFO  [Time-limited test {}] server.Server(415): Started @195515ms
2024-11-09T21:24:08,107 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-09T21:24:08,232 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:08,288 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:24:08,290 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:24:08,291 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:24:08,291 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:24:08,291 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T21:24:08,291 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@280e4ad5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:24:08,292 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6f17b515{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:24:08,384 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6719db8a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/java.io.tmpdir/jetty-localhost-42113-hadoop-hdfs-3_4_1-tests_jar-_-any-5439602956822278248/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:24:08,385 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3e2a30ba{HTTP/1.1, (http/1.1)}{localhost:42113}
2024-11-09T21:24:08,385 INFO  [Time-limited test {}] server.Server(415): Started @195803ms
2024-11-09T21:24:08,385 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:24:08,409 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:24:08,412 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:24:08,413 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:24:08,413 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:24:08,413 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T21:24:08,413 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e428b73{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:24:08,414 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62aa92bd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:24:08,487 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:08,506 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:08,507 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3aa8039f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/java.io.tmpdir/jetty-localhost-37173-hadoop-hdfs-3_4_1-tests_jar-_-any-11870808602997390689/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:24:08,507 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@55244549{HTTP/1.1, (http/1.1)}{localhost:37173}
2024-11-09T21:24:08,507 INFO  [Time-limited test {}] server.Server(415): Started @195926ms
2024-11-09T21:24:08,508 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:24:09,117 WARN  [Thread-1640 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/cluster_d0e49250-96c8-093e-ed32-ea6c84048991/data/data1/current/BP-1494917969-172.17.0.2-1731187447741/current, will proceed with Du for space computation calculation, 
2024-11-09T21:24:09,117 WARN  [Thread-1641 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/cluster_d0e49250-96c8-093e-ed32-ea6c84048991/data/data2/current/BP-1494917969-172.17.0.2-1731187447741/current, will proceed with Du for space computation calculation, 
2024-11-09T21:24:09,137 WARN  [Thread-1604 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:24:09,140 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcb7c71b128da348f with lease ID 0xcd2cb1a96d43c671: Processing first storage report for DS-f4e4047f-7d02-465a-aaa7-eb7e8ad2d689 from datanode DatanodeRegistration(127.0.0.1:45733, datanodeUuid=5c5b9681-d557-4357-ad59-c91cc8dc8620, infoPort=44959, infoSecurePort=0, ipcPort=44851, storageInfo=lv=-57;cid=testClusterID;nsid=910888180;c=1731187447741)
2024-11-09T21:24:09,140 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcb7c71b128da348f with lease ID 0xcd2cb1a96d43c671: from storage DS-f4e4047f-7d02-465a-aaa7-eb7e8ad2d689 node DatanodeRegistration(127.0.0.1:45733, datanodeUuid=5c5b9681-d557-4357-ad59-c91cc8dc8620, infoPort=44959, infoSecurePort=0, ipcPort=44851, storageInfo=lv=-57;cid=testClusterID;nsid=910888180;c=1731187447741), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:24:09,140 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcb7c71b128da348f with lease ID 0xcd2cb1a96d43c671: Processing first storage report for DS-7c1e8728-f26e-48cc-890e-29e2e467bf5c from datanode DatanodeRegistration(127.0.0.1:45733, datanodeUuid=5c5b9681-d557-4357-ad59-c91cc8dc8620, infoPort=44959, infoSecurePort=0, ipcPort=44851, storageInfo=lv=-57;cid=testClusterID;nsid=910888180;c=1731187447741)
2024-11-09T21:24:09,140 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcb7c71b128da348f with lease ID 0xcd2cb1a96d43c671: from storage DS-7c1e8728-f26e-48cc-890e-29e2e467bf5c node DatanodeRegistration(127.0.0.1:45733, datanodeUuid=5c5b9681-d557-4357-ad59-c91cc8dc8620, infoPort=44959, infoSecurePort=0, ipcPort=44851, storageInfo=lv=-57;cid=testClusterID;nsid=910888180;c=1731187447741), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:24:09,233 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:09,255 WARN  [Thread-1651 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/cluster_d0e49250-96c8-093e-ed32-ea6c84048991/data/data3/current/BP-1494917969-172.17.0.2-1731187447741/current, will proceed with Du for space computation calculation, 
2024-11-09T21:24:09,255 WARN  [Thread-1652 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/cluster_d0e49250-96c8-093e-ed32-ea6c84048991/data/data4/current/BP-1494917969-172.17.0.2-1731187447741/current, will proceed with Du for space computation calculation, 
2024-11-09T21:24:09,275 WARN  [Thread-1627 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:24:09,277 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x237322f9066b80d with lease ID 0xcd2cb1a96d43c672: Processing first storage report for DS-49cad724-8631-4a8b-b672-27ffb71f6da3 from datanode DatanodeRegistration(127.0.0.1:44033, datanodeUuid=9926d156-8d06-4340-b0a5-b69cdaf2243e, infoPort=38679, infoSecurePort=0, ipcPort=35041, storageInfo=lv=-57;cid=testClusterID;nsid=910888180;c=1731187447741)
2024-11-09T21:24:09,277 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x237322f9066b80d with lease ID 0xcd2cb1a96d43c672: from storage DS-49cad724-8631-4a8b-b672-27ffb71f6da3 node DatanodeRegistration(127.0.0.1:44033, datanodeUuid=9926d156-8d06-4340-b0a5-b69cdaf2243e, infoPort=38679, infoSecurePort=0, ipcPort=35041, storageInfo=lv=-57;cid=testClusterID;nsid=910888180;c=1731187447741), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:24:09,277 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x237322f9066b80d with lease ID 0xcd2cb1a96d43c672: Processing first storage report for DS-a402b436-b3b2-4b16-bd12-18fc0dc0e246 from datanode DatanodeRegistration(127.0.0.1:44033, datanodeUuid=9926d156-8d06-4340-b0a5-b69cdaf2243e, infoPort=38679, infoSecurePort=0, ipcPort=35041, storageInfo=lv=-57;cid=testClusterID;nsid=910888180;c=1731187447741)
2024-11-09T21:24:09,277 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x237322f9066b80d with lease ID 0xcd2cb1a96d43c672: from storage DS-a402b436-b3b2-4b16-bd12-18fc0dc0e246 node DatanodeRegistration(127.0.0.1:44033, datanodeUuid=9926d156-8d06-4340-b0a5-b69cdaf2243e, infoPort=38679, infoSecurePort=0, ipcPort=35041, storageInfo=lv=-57;cid=testClusterID;nsid=910888180;c=1731187447741), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:24:09,338 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928
2024-11-09T21:24:09,342 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/cluster_d0e49250-96c8-093e-ed32-ea6c84048991/zookeeper_0, clientPort=53181, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/cluster_d0e49250-96c8-093e-ed32-ea6c84048991/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/cluster_d0e49250-96c8-093e-ed32-ea6c84048991/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-09T21:24:09,344 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53181
2024-11-09T21:24:09,344 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:24:09,347 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:24:09,357 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741825_1001 (size=7)
2024-11-09T21:24:09,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741825_1001 (size=7)
2024-11-09T21:24:09,359 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023 with version=8
2024-11-09T21:24:09,359 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/hbase-staging
2024-11-09T21:24:09,361 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d86ff69d6719:0 server-side Connection retries=45
2024-11-09T21:24:09,361 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:24:09,361 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T21:24:09,361 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T21:24:09,361 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:24:09,361 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T21:24:09,361 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-09T21:24:09,361 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T21:24:09,362 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37815
2024-11-09T21:24:09,364 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37815 connecting to ZooKeeper ensemble=127.0.0.1:53181
2024-11-09T21:24:09,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:378150x0, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T21:24:09,401 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37815-0x10122c5b0c20000 connected
2024-11-09T21:24:09,465 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:24:09,470 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:24:09,472 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:24:09,472 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023, hbase.cluster.distributed=false
2024-11-09T21:24:09,474 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T21:24:09,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37815
2024-11-09T21:24:09,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37815
2024-11-09T21:24:09,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37815
2024-11-09T21:24:09,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37815
2024-11-09T21:24:09,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37815
2024-11-09T21:24:09,487 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:09,492 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d86ff69d6719:0 server-side Connection retries=45
2024-11-09T21:24:09,492 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:24:09,492 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T21:24:09,493 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T21:24:09,493 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:24:09,493 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T21:24:09,493 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-09T21:24:09,493 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T21:24:09,494 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37125
2024-11-09T21:24:09,495 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37125 connecting to ZooKeeper ensemble=127.0.0.1:53181
2024-11-09T21:24:09,495 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:24:09,497 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:24:09,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:371250x0, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T21:24:09,506 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:371250x0, quorum=127.0.0.1:53181, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:24:09,506 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37125-0x10122c5b0c20001 connected
2024-11-09T21:24:09,506 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-09T21:24:09,507 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-09T21:24:09,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:09,508 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-09T21:24:09,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T21:24:09,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37125
2024-11-09T21:24:09,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37125
2024-11-09T21:24:09,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37125
2024-11-09T21:24:09,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37125
2024-11-09T21:24:09,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37125
2024-11-09T21:24:09,520 DEBUG [M:0;d86ff69d6719:37815 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d86ff69d6719:37815
2024-11-09T21:24:09,521 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d86ff69d6719,37815,1731187449361
2024-11-09T21:24:09,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:24:09,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:24:09,531 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d86ff69d6719,37815,1731187449361
2024-11-09T21:24:09,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:24:09,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-09T21:24:09,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:24:09,540 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-09T21:24:09,540 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d86ff69d6719,37815,1731187449361 from backup master directory
2024-11-09T21:24:09,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d86ff69d6719,37815,1731187449361
2024-11-09T21:24:09,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:24:09,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:24:09,548 WARN  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T21:24:09,548 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d86ff69d6719,37815,1731187449361
2024-11-09T21:24:09,554 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/hbase.id] with ID: 92f6cd97-f3bb-4886-9f27-acf71dff871a
2024-11-09T21:24:09,554 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/.tmp/hbase.id
2024-11-09T21:24:09,563 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741826_1002 (size=42)
2024-11-09T21:24:09,563 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-09T21:24:09,563 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-09T21:24:09,564 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-09T21:24:09,564 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-09T21:24:09,564 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741826_1002 (size=42)
2024-11-09T21:24:09,565 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/.tmp/hbase.id]:[hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/hbase.id]
2024-11-09T21:24:09,575 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:24:09,575 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-09T21:24:09,577 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-09T21:24:09,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:24:09,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:24:09,591 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741827_1003 (size=196)
2024-11-09T21:24:09,592 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741827_1003 (size=196)
2024-11-09T21:24:09,592 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-09T21:24:09,593 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-09T21:24:09,593 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:24:09,599 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741828_1004 (size=1189)
2024-11-09T21:24:09,599 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741828_1004 (size=1189)
2024-11-09T21:24:09,599 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store
2024-11-09T21:24:09,605 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741829_1005 (size=34)
2024-11-09T21:24:09,605 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741829_1005 (size=34)
2024-11-09T21:24:09,605 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:24:09,605 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-09T21:24:09,605 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:24:09,605 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:24:09,606 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-09T21:24:09,606 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:24:09,606 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:24:09,606 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731187449605Disabling compacts and flushes for region at 1731187449605Disabling writes for close at 1731187449606 (+1 ms)Writing region close event to WAL at 1731187449606Closed at 1731187449606
2024-11-09T21:24:09,606 WARN  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/.initializing
2024-11-09T21:24:09,606 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/WALs/d86ff69d6719,37815,1731187449361
2024-11-09T21:24:09,609 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C37815%2C1731187449361, suffix=, logDir=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/WALs/d86ff69d6719,37815,1731187449361, archiveDir=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/oldWALs, maxLogs=10
2024-11-09T21:24:09,609 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C37815%2C1731187449361.1731187449609
2024-11-09T21:24:09,613 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/WALs/d86ff69d6719,37815,1731187449361/d86ff69d6719%2C37815%2C1731187449361.1731187449609
2024-11-09T21:24:09,614 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44959:44959),(127.0.0.1/127.0.0.1:38679:38679)]
2024-11-09T21:24:09,615 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:24:09,615 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:24:09,615 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:24:09,615 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:24:09,617 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:24:09,618 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-09T21:24:09,618 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:24:09,618 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:24:09,618 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:24:09,619 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-09T21:24:09,620 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:24:09,620 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:24:09,620 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:24:09,621 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-09T21:24:09,621 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:24:09,622 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:24:09,622 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:24:09,623 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-09T21:24:09,623 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:24:09,623 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:24:09,623 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:24:09,624 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:24:09,624 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:24:09,625 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:24:09,625 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:24:09,626 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-09T21:24:09,627 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:24:09,629 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:24:09,630 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=856919, jitterRate=0.08962957561016083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-09T21:24:09,630 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731187449615Initializing all the Stores at 1731187449616 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187449616Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187449616Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187449616Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187449616Cleaning up temporary data from old regions at 1731187449625 (+9 ms)Region opened successfully at 1731187449630 (+5 ms)
2024-11-09T21:24:09,631 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-09T21:24:09,634 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5efa317d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0
2024-11-09T21:24:09,635 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-09T21:24:09,635 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-09T21:24:09,635 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-09T21:24:09,635 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-09T21:24:09,635 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-09T21:24:09,636 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-09T21:24:09,636 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-09T21:24:09,637 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-09T21:24:09,638 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-09T21:24:09,647 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-09T21:24:09,647 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-09T21:24:09,648 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-09T21:24:09,655 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-09T21:24:09,656 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-09T21:24:09,657 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-09T21:24:09,664 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-09T21:24:09,665 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-09T21:24:09,672 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-09T21:24:09,674 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-09T21:24:09,681 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-09T21:24:09,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-09T21:24:09,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-09T21:24:09,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:24:09,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:24:09,690 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d86ff69d6719,37815,1731187449361, sessionid=0x10122c5b0c20000, setting cluster-up flag (Was=false)
2024-11-09T21:24:09,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:24:09,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:24:09,731 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-09T21:24:09,733 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d86ff69d6719,37815,1731187449361
2024-11-09T21:24:09,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:24:09,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:24:09,772 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-09T21:24:09,775 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d86ff69d6719,37815,1731187449361
2024-11-09T21:24:09,777 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-09T21:24:09,780 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-09T21:24:09,781 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-09T21:24:09,781 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-09T21:24:09,781 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d86ff69d6719,37815,1731187449361
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-09T21:24:09,783 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:24:09,783 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:24:09,783 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:24:09,783 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:24:09,783 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d86ff69d6719:0, corePoolSize=10, maxPoolSize=10
2024-11-09T21:24:09,783 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:24:09,783 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2
2024-11-09T21:24:09,783 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:24:09,784 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731187479784
2024-11-09T21:24:09,784 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-09T21:24:09,784 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-09T21:24:09,785 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-09T21:24:09,785 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-09T21:24:09,785 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-09T21:24:09,785 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-09T21:24:09,785 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:09,785 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-09T21:24:09,785 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-09T21:24:09,785 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:24:09,785 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-09T21:24:09,785 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-09T21:24:09,786 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-09T21:24:09,786 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-09T21:24:09,786 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187449786,5,FailOnTimeoutGroup]
2024-11-09T21:24:09,786 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187449786,5,FailOnTimeoutGroup]
2024-11-09T21:24:09,786 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:09,786 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-09T21:24:09,786 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:09,786 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:09,787 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:24:09,787 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-09T21:24:09,794 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741831_1007 (size=1321)
2024-11-09T21:24:09,794 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741831_1007 (size=1321)
2024-11-09T21:24:09,795 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-09T21:24:09,796 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023
2024-11-09T21:24:09,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741832_1008 (size=32)
2024-11-09T21:24:09,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741832_1008 (size=32)
2024-11-09T21:24:09,802 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:24:09,803 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-09T21:24:09,804 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-09T21:24:09,804 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:24:09,805 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:24:09,805 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-09T21:24:09,806 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-09T21:24:09,806 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:24:09,806 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:24:09,806 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-09T21:24:09,807 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-09T21:24:09,807 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:24:09,808 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:24:09,808 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-09T21:24:09,809 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-09T21:24:09,809 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:24:09,809 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:24:09,809 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-09T21:24:09,810 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740
2024-11-09T21:24:09,810 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740
2024-11-09T21:24:09,811 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-09T21:24:09,811 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-09T21:24:09,812 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer(746): ClusterId : 92f6cd97-f3bb-4886-9f27-acf71dff871a
2024-11-09T21:24:09,812 DEBUG [RS:0;d86ff69d6719:37125 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-09T21:24:09,812 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-09T21:24:09,813 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-09T21:24:09,814 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:24:09,815 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=790865, jitterRate=0.005637168884277344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-09T21:24:09,815 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731187449802Initializing all the Stores at 1731187449803 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187449803Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187449803Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187449803Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187449803Cleaning up temporary data from old regions at 1731187449811 (+8 ms)Region opened successfully at 1731187449815 (+4 ms)
2024-11-09T21:24:09,815 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-09T21:24:09,815 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-09T21:24:09,816 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-09T21:24:09,816 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-09T21:24:09,816 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-09T21:24:09,816 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-09T21:24:09,816 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731187449815Disabling compacts and flushes for region at 1731187449815Disabling writes for close at 1731187449816 (+1 ms)Writing region close event to WAL at 1731187449816Closed at 1731187449816
2024-11-09T21:24:09,817 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:24:09,817 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-09T21:24:09,817 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-09T21:24:09,819 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-09T21:24:09,820 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-09T21:24:09,823 DEBUG [RS:0;d86ff69d6719:37125 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-09T21:24:09,823 DEBUG [RS:0;d86ff69d6719:37125 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-09T21:24:09,832 DEBUG [RS:0;d86ff69d6719:37125 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-09T21:24:09,832 DEBUG [RS:0;d86ff69d6719:37125 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15400358, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0
2024-11-09T21:24:09,847 DEBUG [RS:0;d86ff69d6719:37125 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d86ff69d6719:37125
2024-11-09T21:24:09,847 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-09T21:24:09,847 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-09T21:24:09,847 DEBUG [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-09T21:24:09,847 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer(2659): reportForDuty to master=d86ff69d6719,37815,1731187449361 with port=37125, startcode=1731187449492
2024-11-09T21:24:09,848 DEBUG [RS:0;d86ff69d6719:37125 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-09T21:24:09,849 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35191, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-11-09T21:24:09,850 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37815 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d86ff69d6719,37125,1731187449492
2024-11-09T21:24:09,850 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37815 {}] master.ServerManager(517): Registering regionserver=d86ff69d6719,37125,1731187449492
2024-11-09T21:24:09,851 DEBUG [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023
2024-11-09T21:24:09,851 DEBUG [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35747
2024-11-09T21:24:09,851 DEBUG [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-09T21:24:09,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T21:24:09,856 DEBUG [RS:0;d86ff69d6719:37125 {}] zookeeper.ZKUtil(111): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d86ff69d6719,37125,1731187449492
2024-11-09T21:24:09,856 WARN  [RS:0;d86ff69d6719:37125 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T21:24:09,856 INFO  [RS:0;d86ff69d6719:37125 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:24:09,856 DEBUG [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/WALs/d86ff69d6719,37125,1731187449492
2024-11-09T21:24:09,857 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d86ff69d6719,37125,1731187449492]
2024-11-09T21:24:09,859 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-09T21:24:09,861 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-09T21:24:09,861 INFO  [RS:0;d86ff69d6719:37125 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-09T21:24:09,861 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:09,861 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-09T21:24:09,862 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-09T21:24:09,862 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:09,862 DEBUG [RS:0;d86ff69d6719:37125 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:24:09,862 DEBUG [RS:0;d86ff69d6719:37125 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:24:09,862 DEBUG [RS:0;d86ff69d6719:37125 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:24:09,862 DEBUG [RS:0;d86ff69d6719:37125 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:24:09,862 DEBUG [RS:0;d86ff69d6719:37125 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:24:09,863 DEBUG [RS:0;d86ff69d6719:37125 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2
2024-11-09T21:24:09,863 DEBUG [RS:0;d86ff69d6719:37125 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:24:09,863 DEBUG [RS:0;d86ff69d6719:37125 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:24:09,863 DEBUG [RS:0;d86ff69d6719:37125 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:24:09,863 DEBUG [RS:0;d86ff69d6719:37125 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:24:09,863 DEBUG [RS:0;d86ff69d6719:37125 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:24:09,863 DEBUG [RS:0;d86ff69d6719:37125 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:24:09,863 DEBUG [RS:0;d86ff69d6719:37125 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3
2024-11-09T21:24:09,863 DEBUG [RS:0;d86ff69d6719:37125 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3
2024-11-09T21:24:09,863 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:09,863 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:09,863 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:09,863 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:09,864 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:09,864 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,37125,1731187449492-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T21:24:09,880 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-09T21:24:09,880 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,37125,1731187449492-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:09,880 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:09,880 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.Replication(171): d86ff69d6719,37125,1731187449492 started
2024-11-09T21:24:09,893 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:09,893 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer(1482): Serving as d86ff69d6719,37125,1731187449492, RpcServer on d86ff69d6719/172.17.0.2:37125, sessionid=0x10122c5b0c20001
2024-11-09T21:24:09,893 DEBUG [RS:0;d86ff69d6719:37125 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-09T21:24:09,893 DEBUG [RS:0;d86ff69d6719:37125 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d86ff69d6719,37125,1731187449492
2024-11-09T21:24:09,893 DEBUG [RS:0;d86ff69d6719:37125 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,37125,1731187449492'
2024-11-09T21:24:09,893 DEBUG [RS:0;d86ff69d6719:37125 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-09T21:24:09,894 DEBUG [RS:0;d86ff69d6719:37125 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-09T21:24:09,894 DEBUG [RS:0;d86ff69d6719:37125 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-09T21:24:09,894 DEBUG [RS:0;d86ff69d6719:37125 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-09T21:24:09,894 DEBUG [RS:0;d86ff69d6719:37125 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d86ff69d6719,37125,1731187449492
2024-11-09T21:24:09,894 DEBUG [RS:0;d86ff69d6719:37125 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,37125,1731187449492'
2024-11-09T21:24:09,894 DEBUG [RS:0;d86ff69d6719:37125 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-09T21:24:09,895 DEBUG [RS:0;d86ff69d6719:37125 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-09T21:24:09,895 DEBUG [RS:0;d86ff69d6719:37125 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-09T21:24:09,895 INFO  [RS:0;d86ff69d6719:37125 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-09T21:24:09,895 INFO  [RS:0;d86ff69d6719:37125 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-09T21:24:09,970 WARN  [d86ff69d6719:37815 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-09T21:24:09,997 INFO  [RS:0;d86ff69d6719:37125 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C37125%2C1731187449492, suffix=, logDir=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/WALs/d86ff69d6719,37125,1731187449492, archiveDir=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/oldWALs, maxLogs=32
2024-11-09T21:24:09,998 INFO  [RS:0;d86ff69d6719:37125 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C37125%2C1731187449492.1731187449998
2024-11-09T21:24:10,005 INFO  [RS:0;d86ff69d6719:37125 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/WALs/d86ff69d6719,37125,1731187449492/d86ff69d6719%2C37125%2C1731187449492.1731187449998
2024-11-09T21:24:10,006 DEBUG [RS:0;d86ff69d6719:37125 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44959:44959),(127.0.0.1/127.0.0.1:38679:38679)]
2024-11-09T21:24:10,220 DEBUG [d86ff69d6719:37815 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-09T21:24:10,221 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d86ff69d6719,37125,1731187449492
2024-11-09T21:24:10,225 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d86ff69d6719,37125,1731187449492, state=OPENING
2024-11-09T21:24:10,234 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:10,264 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-09T21:24:10,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:24:10,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:24:10,275 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-09T21:24:10,275 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:24:10,275 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:24:10,275 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d86ff69d6719,37125,1731187449492}]
2024-11-09T21:24:10,432 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-09T21:24:10,436 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33881, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-09T21:24:10,444 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-09T21:24:10,444 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:24:10,447 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C37125%2C1731187449492.meta, suffix=.meta, logDir=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/WALs/d86ff69d6719,37125,1731187449492, archiveDir=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/oldWALs, maxLogs=32
2024-11-09T21:24:10,448 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C37125%2C1731187449492.meta.1731187450447.meta
2024-11-09T21:24:10,454 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/WALs/d86ff69d6719,37125,1731187449492/d86ff69d6719%2C37125%2C1731187449492.meta.1731187450447.meta
2024-11-09T21:24:10,455 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38679:38679),(127.0.0.1/127.0.0.1:44959:44959)]
2024-11-09T21:24:10,456 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:24:10,456 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-09T21:24:10,456 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-09T21:24:10,457 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-09T21:24:10,457 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-09T21:24:10,457 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:24:10,457 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-09T21:24:10,457 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-09T21:24:10,458 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-09T21:24:10,460 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-09T21:24:10,460 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:24:10,460 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:24:10,460 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-09T21:24:10,461 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-09T21:24:10,461 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:24:10,462 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:24:10,462 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-09T21:24:10,463 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-09T21:24:10,463 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:24:10,464 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:24:10,464 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-09T21:24:10,465 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-09T21:24:10,465 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:24:10,465 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:24:10,466 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-09T21:24:10,466 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740
2024-11-09T21:24:10,468 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740
2024-11-09T21:24:10,469 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-09T21:24:10,469 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-09T21:24:10,470 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-09T21:24:10,472 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-09T21:24:10,473 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=799800, jitterRate=0.016998961567878723}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-09T21:24:10,473 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-09T21:24:10,474 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731187450457Writing region info on filesystem at 1731187450457Initializing all the Stores at 1731187450458 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187450458Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187450458Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187450458Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187450458Cleaning up temporary data from old regions at 1731187450469 (+11 ms)Running coprocessor post-open hooks at 1731187450473 (+4 ms)Region opened successfully at 1731187450474 (+1 ms)
2024-11-09T21:24:10,475 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731187450431
2024-11-09T21:24:10,478 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-09T21:24:10,478 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-09T21:24:10,479 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,37125,1731187449492
2024-11-09T21:24:10,481 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d86ff69d6719,37125,1731187449492, state=OPEN
2024-11-09T21:24:10,488 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:10,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:10,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-09T21:24:10,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-09T21:24:10,517 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d86ff69d6719,37125,1731187449492
2024-11-09T21:24:10,517 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:24:10,517 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:24:10,521 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-09T21:24:10,522 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d86ff69d6719,37125,1731187449492 in 242 msec
2024-11-09T21:24:10,527 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-09T21:24:10,527 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 705 msec
2024-11-09T21:24:10,529 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:24:10,529 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-09T21:24:10,531 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-09T21:24:10,531 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,37125,1731187449492, seqNum=-1]
2024-11-09T21:24:10,532 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T21:24:10,533 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45073, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T21:24:10,539 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 759 msec
2024-11-09T21:24:10,539 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731187450539, completionTime=-1
2024-11-09T21:24:10,539 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-09T21:24:10,539 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-09T21:24:10,541 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-09T21:24:10,541 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731187510541
2024-11-09T21:24:10,541 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731187570541
2024-11-09T21:24:10,541 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-11-09T21:24:10,541 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,37815,1731187449361-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:10,541 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,37815,1731187449361-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:10,541 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,37815,1731187449361-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:10,541 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d86ff69d6719:37815, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:10,541 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:10,542 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:10,543 DEBUG [master/d86ff69d6719:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-09T21:24:10,545 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.997sec
2024-11-09T21:24:10,545 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-09T21:24:10,545 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-09T21:24:10,545 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-09T21:24:10,545 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-09T21:24:10,545 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-09T21:24:10,546 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,37815,1731187449361-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T21:24:10,546 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,37815,1731187449361-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-09T21:24:10,548 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-09T21:24:10,548 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-09T21:24:10,548 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,37815,1731187449361-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:10,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dcbd726, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:24:10,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d86ff69d6719,37815,-1 for getting cluster id
2024-11-09T21:24:10,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-09T21:24:10,615 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '92f6cd97-f3bb-4886-9f27-acf71dff871a'
2024-11-09T21:24:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-09T21:24:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "92f6cd97-f3bb-4886-9f27-acf71dff871a"

2024-11-09T21:24:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@137698c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:24:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d86ff69d6719,37815,-1]
2024-11-09T21:24:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-09T21:24:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:24:10,619 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-09T21:24:10,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d9a0032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:24:10,621 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-09T21:24:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,37125,1731187449492, seqNum=-1]
2024-11-09T21:24:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T21:24:10,625 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T21:24:10,626 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d86ff69d6719,37815,1731187449361
2024-11-09T21:24:10,626 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:24:10,629 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-09T21:24:10,629 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-09T21:24:10,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is d86ff69d6719,37815,1731187449361
2024-11-09T21:24:10,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@d7544a5
2024-11-09T21:24:10,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-09T21:24:10,631 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-09T21:24:10,632 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-09T21:24:10,632 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-09T21:24:10,632 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-09T21:24:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-09T21:24:10,635 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-09T21:24:10,635 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:24:10,635 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-11-09T21:24:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-09T21:24:10,637 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-09T21:24:10,645 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741835_1011 (size=405)
2024-11-09T21:24:10,645 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741835_1011 (size=405)
2024-11-09T21:24:10,647 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7aadedad35e084e85fd0104228cb95a3, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023
2024-11-09T21:24:10,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741836_1012 (size=88)
2024-11-09T21:24:10,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741836_1012 (size=88)
2024-11-09T21:24:10,655 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:24:10,655 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 7aadedad35e084e85fd0104228cb95a3, disabling compactions & flushes
2024-11-09T21:24:10,655 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:10,655 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:10,655 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3. after waiting 0 ms
2024-11-09T21:24:10,655 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:10,655 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:10,655 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7aadedad35e084e85fd0104228cb95a3:
Waiting for close lock at 1731187450655Disabling compacts and flushes for region at 1731187450655Disabling writes for close at 1731187450655Writing region close event to WAL at 1731187450655Closed at 1731187450655
2024-11-09T21:24:10,657 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-09T21:24:10,657 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731187450657"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731187450657"}]},"ts":"1731187450657"}
2024-11-09T21:24:10,659 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-09T21:24:10,660 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-09T21:24:10,661 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187450660"}]},"ts":"1731187450660"}
2024-11-09T21:24:10,663 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-11-09T21:24:10,663 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=7aadedad35e084e85fd0104228cb95a3, ASSIGN}]
2024-11-09T21:24:10,664 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=7aadedad35e084e85fd0104228cb95a3, ASSIGN
2024-11-09T21:24:10,665 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=7aadedad35e084e85fd0104228cb95a3, ASSIGN; state=OFFLINE, location=d86ff69d6719,37125,1731187449492; forceNewPlan=false, retain=false
2024-11-09T21:24:10,816 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7aadedad35e084e85fd0104228cb95a3, regionState=OPENING, regionLocation=d86ff69d6719,37125,1731187449492
2024-11-09T21:24:10,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=7aadedad35e084e85fd0104228cb95a3, ASSIGN because future has completed
2024-11-09T21:24:10,819 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aadedad35e084e85fd0104228cb95a3, server=d86ff69d6719,37125,1731187449492}]
2024-11-09T21:24:10,979 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:10,980 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7aadedad35e084e85fd0104228cb95a3, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:24:10,980 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 7aadedad35e084e85fd0104228cb95a3
2024-11-09T21:24:10,981 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:24:10,981 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7aadedad35e084e85fd0104228cb95a3
2024-11-09T21:24:10,981 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7aadedad35e084e85fd0104228cb95a3
2024-11-09T21:24:10,984 INFO  [StoreOpener-7aadedad35e084e85fd0104228cb95a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 7aadedad35e084e85fd0104228cb95a3 
2024-11-09T21:24:10,987 INFO  [StoreOpener-7aadedad35e084e85fd0104228cb95a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7aadedad35e084e85fd0104228cb95a3 columnFamilyName info
2024-11-09T21:24:10,987 DEBUG [StoreOpener-7aadedad35e084e85fd0104228cb95a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:24:10,987 INFO  [StoreOpener-7aadedad35e084e85fd0104228cb95a3-1 {}] regionserver.HStore(327): Store=7aadedad35e084e85fd0104228cb95a3/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:24:10,987 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7aadedad35e084e85fd0104228cb95a3
2024-11-09T21:24:10,988 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3
2024-11-09T21:24:10,988 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3
2024-11-09T21:24:10,989 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7aadedad35e084e85fd0104228cb95a3
2024-11-09T21:24:10,989 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7aadedad35e084e85fd0104228cb95a3
2024-11-09T21:24:10,990 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7aadedad35e084e85fd0104228cb95a3
2024-11-09T21:24:10,992 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:24:10,992 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7aadedad35e084e85fd0104228cb95a3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=863978, jitterRate=0.09860606491565704}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-09T21:24:10,993 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7aadedad35e084e85fd0104228cb95a3
2024-11-09T21:24:10,993 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7aadedad35e084e85fd0104228cb95a3:
Running coprocessor pre-open hook at 1731187450981Writing region info on filesystem at 1731187450981Initializing all the Stores at 1731187450983 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187450983Cleaning up temporary data from old regions at 1731187450989 (+6 ms)Running coprocessor post-open hooks at 1731187450993 (+4 ms)Region opened successfully at 1731187450993
2024-11-09T21:24:10,994 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3., pid=6, masterSystemTime=1731187450972
2024-11-09T21:24:10,996 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:10,996 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:10,997 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7aadedad35e084e85fd0104228cb95a3, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,37125,1731187449492
2024-11-09T21:24:10,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7aadedad35e084e85fd0104228cb95a3, server=d86ff69d6719,37125,1731187449492 because future has completed
2024-11-09T21:24:11,004 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-09T21:24:11,004 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7aadedad35e084e85fd0104228cb95a3, server=d86ff69d6719,37125,1731187449492 in 182 msec
2024-11-09T21:24:11,006 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-09T21:24:11,006 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=7aadedad35e084e85fd0104228cb95a3, ASSIGN in 341 msec
2024-11-09T21:24:11,008 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-09T21:24:11,008 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187451008"}]},"ts":"1731187451008"}
2024-11-09T21:24:11,010 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-11-09T21:24:11,011 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-09T21:24:11,013 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 379 msec
2024-11-09T21:24:11,235 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:11,489 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:11,508 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:12,237 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:12,490 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:12,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:13,238 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:13,491 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:13,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:14,239 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:14,492 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:14,510 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:15,070 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-09T21:24:15,071 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:15,072 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:15,072 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:15,073 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:15,073 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:15,074 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:15,096 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:15,096 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:15,096 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:15,097 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:15,097 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:15,097 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:15,100 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:15,100 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:15,100 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:15,103 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:24:15,240 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:15,494 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:15,512 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:15,860 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-09T21:24:15,862 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-11-09T21:24:16,241 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:16,495 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:16,513 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:17,242 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:17,496 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:17,514 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:18,244 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:18,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:18,515 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:19,245 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:19,499 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:19,516 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:19,563 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-09T21:24:19,564 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-09T21:24:19,566 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-09T21:24:19,566 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-09T21:24:19,567 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-09T21:24:19,567 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-09T21:24:19,569 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-09T21:24:19,569 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-11-09T21:24:20,246 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:20,500 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:20,517 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:20,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-09T21:24:20,685 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-09T21:24:20,686 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-11-09T21:24:20,693 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-09T21:24:20,693 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:20,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3., hostname=d86ff69d6719,37125,1731187449492, seqNum=2]
2024-11-09T21:24:20,705 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-09T21:24:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-09T21:24:20,710 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-09T21:24:20,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-09T21:24:20,712 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-09T21:24:20,713 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-09T21:24:20,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37125 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-11-09T21:24:20,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:20,884 INFO  [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 7aadedad35e084e85fd0104228cb95a3 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-09T21:24:20,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/0ce318ce8f4940799e6cf1564bc3fd6d is 1080, key is row0001/info:/1731187460697/Put/seqid=0
2024-11-09T21:24:20,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741837_1013 (size=6033)
2024-11-09T21:24:20,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741837_1013 (size=6033)
2024-11-09T21:24:20,905 INFO  [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/0ce318ce8f4940799e6cf1564bc3fd6d
2024-11-09T21:24:20,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/0ce318ce8f4940799e6cf1564bc3fd6d as hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/0ce318ce8f4940799e6cf1564bc3fd6d
2024-11-09T21:24:20,918 INFO  [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/0ce318ce8f4940799e6cf1564bc3fd6d, entries=1, sequenceid=5, filesize=5.9 K
2024-11-09T21:24:20,919 INFO  [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7aadedad35e084e85fd0104228cb95a3 in 35ms, sequenceid=5, compaction requested=false
2024-11-09T21:24:20,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 7aadedad35e084e85fd0104228cb95a3:

2024-11-09T21:24:20,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:20,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-09T21:24:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-09T21:24:20,927 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-09T21:24:20,927 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 211 msec
2024-11-09T21:24:20,930 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 222 msec
2024-11-09T21:24:21,248 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:21,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:21,518 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:22,249 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:22,503 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:22,519 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:23,250 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:23,504 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:23,520 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:24,251 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:24,505 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:24,521 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:25,252 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:25,506 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:25,522 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:26,254 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:26,508 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:26,523 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:27,255 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:27,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:27,525 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:28,256 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:28,510 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:28,511 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735 after 68078ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor191.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:24:28,526 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:28,527 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta after 68087ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor191.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-09T21:24:29,258 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:29,512 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:29,528 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:30,259 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:30,514 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:30,529 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:30,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-09T21:24:30,785 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-09T21:24:30,792 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-09T21:24:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-09T21:24:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-09T21:24:30,796 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-09T21:24:30,798 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-09T21:24:30,798 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-09T21:24:30,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37125 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-11-09T21:24:30,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:30,955 INFO  [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 7aadedad35e084e85fd0104228cb95a3 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-09T21:24:30,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/a0cf87848f984ac1acbefff355a3bd17 is 1080, key is row0002/info:/1731187470788/Put/seqid=0
2024-11-09T21:24:30,973 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741838_1014 (size=6033)
2024-11-09T21:24:30,973 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741838_1014 (size=6033)
2024-11-09T21:24:30,974 INFO  [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/a0cf87848f984ac1acbefff355a3bd17
2024-11-09T21:24:30,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/a0cf87848f984ac1acbefff355a3bd17 as hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/a0cf87848f984ac1acbefff355a3bd17
2024-11-09T21:24:30,987 INFO  [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/a0cf87848f984ac1acbefff355a3bd17, entries=1, sequenceid=9, filesize=5.9 K
2024-11-09T21:24:30,989 INFO  [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7aadedad35e084e85fd0104228cb95a3 in 34ms, sequenceid=9, compaction requested=false
2024-11-09T21:24:30,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 7aadedad35e084e85fd0104228cb95a3:

2024-11-09T21:24:30,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:30,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-09T21:24:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-11-09T21:24:30,993 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-09T21:24:30,993 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 193 msec
2024-11-09T21:24:30,996 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 202 msec
2024-11-09T21:24:31,260 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:31,515 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:31,530 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:32,261 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:32,516 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:32,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:33,263 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:33,517 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:33,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:34,264 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:34,519 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:34,532 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:35,265 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:35,520 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:35,533 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:36,266 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:36,521 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:36,534 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:37,268 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:37,522 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:37,535 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:38,269 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:38,524 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:38,536 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:39,271 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:39,337 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T21:24:39,525 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:39,537 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:40,272 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:40,527 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:40,538 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:40,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-09T21:24:40,805 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-09T21:24:40,813 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C37125%2C1731187449492.1731187480812
2024-11-09T21:24:40,858 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:40,858 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:40,858 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:40,858 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:40,858 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:40,859 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/WALs/d86ff69d6719,37125,1731187449492/d86ff69d6719%2C37125%2C1731187449492.1731187449998 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/WALs/d86ff69d6719,37125,1731187449492/d86ff69d6719%2C37125%2C1731187449492.1731187480812
2024-11-09T21:24:40,860 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38679:38679),(127.0.0.1/127.0.0.1:44959:44959)]
2024-11-09T21:24:40,860 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/WALs/d86ff69d6719,37125,1731187449492/d86ff69d6719%2C37125%2C1731187449492.1731187449998 is not closed yet, will try archiving it next time
2024-11-09T21:24:40,861 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-09T21:24:40,861 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741833_1009 (size=5546)
2024-11-09T21:24:40,862 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741833_1009 (size=5546)
2024-11-09T21:24:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-09T21:24:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-09T21:24:40,864 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-09T21:24:40,866 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-09T21:24:40,866 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-09T21:24:41,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37125 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-11-09T21:24:41,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:41,022 INFO  [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 7aadedad35e084e85fd0104228cb95a3 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-09T21:24:41,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/f4acac1a9b494ee1a3100e42df13ee64 is 1080, key is row0003/info:/1731187480808/Put/seqid=0
2024-11-09T21:24:41,038 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741840_1016 (size=6033)
2024-11-09T21:24:41,038 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741840_1016 (size=6033)
2024-11-09T21:24:41,039 INFO  [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/f4acac1a9b494ee1a3100e42df13ee64
2024-11-09T21:24:41,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/f4acac1a9b494ee1a3100e42df13ee64 as hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/f4acac1a9b494ee1a3100e42df13ee64
2024-11-09T21:24:41,053 INFO  [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/f4acac1a9b494ee1a3100e42df13ee64, entries=1, sequenceid=13, filesize=5.9 K
2024-11-09T21:24:41,054 INFO  [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7aadedad35e084e85fd0104228cb95a3 in 33ms, sequenceid=13, compaction requested=true
2024-11-09T21:24:41,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 7aadedad35e084e85fd0104228cb95a3:

2024-11-09T21:24:41,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:41,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-09T21:24:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-11-09T21:24:41,058 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-09T21:24:41,059 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 190 msec
2024-11-09T21:24:41,061 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 198 msec
2024-11-09T21:24:41,273 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:41,528 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:41,539 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:42,274 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:42,529 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:42,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:43,275 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:43,530 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:43,541 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:44,276 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:44,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:44,542 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:45,277 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:45,532 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:45,543 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:46,278 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:46,534 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:46,544 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:47,279 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:47,535 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:47,545 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:48,281 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:48,536 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:48,546 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:49,282 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:49,537 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:49,547 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:50,283 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:50,539 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:50,548 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:50,655 INFO  [master/d86ff69d6719:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-09T21:24:50,655 INFO  [master/d86ff69d6719:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-09T21:24:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-09T21:24:50,925 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-09T21:24:50,925 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-09T21:24:50,929 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-09T21:24:50,930 DEBUG [Time-limited test {}] regionserver.HStore(1541): 7aadedad35e084e85fd0104228cb95a3/info is initiating minor compaction (all files)
2024-11-09T21:24:50,930 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-09T21:24:50,930 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:24:50,931 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 7aadedad35e084e85fd0104228cb95a3/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:50,931 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/0ce318ce8f4940799e6cf1564bc3fd6d, hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/a0cf87848f984ac1acbefff355a3bd17, hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/f4acac1a9b494ee1a3100e42df13ee64] into tmpdir=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp, totalSize=17.7 K
2024-11-09T21:24:50,932 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0ce318ce8f4940799e6cf1564bc3fd6d, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731187460697
2024-11-09T21:24:50,932 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a0cf87848f984ac1acbefff355a3bd17, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1731187470788
2024-11-09T21:24:50,933 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f4acac1a9b494ee1a3100e42df13ee64, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731187480808
2024-11-09T21:24:50,946 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 7aadedad35e084e85fd0104228cb95a3#info#compaction#45 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-09T21:24:50,947 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/042982fa319e419b9490fcd70405e93c is 1080, key is row0001/info:/1731187460697/Put/seqid=0
2024-11-09T21:24:50,951 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741841_1017 (size=8296)
2024-11-09T21:24:50,951 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741841_1017 (size=8296)
2024-11-09T21:24:50,957 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/042982fa319e419b9490fcd70405e93c as hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/042982fa319e419b9490fcd70405e93c
2024-11-09T21:24:50,964 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7aadedad35e084e85fd0104228cb95a3/info of 7aadedad35e084e85fd0104228cb95a3 into 042982fa319e419b9490fcd70405e93c(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-09T21:24:50,964 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 7aadedad35e084e85fd0104228cb95a3:

2024-11-09T21:24:50,966 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C37125%2C1731187449492.1731187490966
2024-11-09T21:24:50,972 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:50,972 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:50,972 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:50,972 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:50,972 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:24:50,973 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/WALs/d86ff69d6719,37125,1731187449492/d86ff69d6719%2C37125%2C1731187449492.1731187480812 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/WALs/d86ff69d6719,37125,1731187449492/d86ff69d6719%2C37125%2C1731187449492.1731187490966
2024-11-09T21:24:50,973 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44959:44959),(127.0.0.1/127.0.0.1:38679:38679)]
2024-11-09T21:24:50,973 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/WALs/d86ff69d6719,37125,1731187449492/d86ff69d6719%2C37125%2C1731187449492.1731187480812 is not closed yet, will try archiving it next time
2024-11-09T21:24:50,974 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/WALs/d86ff69d6719,37125,1731187449492/d86ff69d6719%2C37125%2C1731187449492.1731187449998 to hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/oldWALs/d86ff69d6719%2C37125%2C1731187449492.1731187449998
2024-11-09T21:24:50,974 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741839_1015 (size=2520)
2024-11-09T21:24:50,974 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-09T21:24:50,975 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741839_1015 (size=2520)
2024-11-09T21:24:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-09T21:24:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-09T21:24:50,977 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-09T21:24:50,977 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-09T21:24:50,977 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-09T21:24:51,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37125 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-11-09T21:24:51,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:51,132 INFO  [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 7aadedad35e084e85fd0104228cb95a3 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-09T21:24:51,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/b3de371ab3eb4a66944c7863bbeb5204 is 1080, key is row0000/info:/1731187490965/Put/seqid=0
2024-11-09T21:24:51,148 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741843_1019 (size=6033)
2024-11-09T21:24:51,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741843_1019 (size=6033)
2024-11-09T21:24:51,149 INFO  [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/b3de371ab3eb4a66944c7863bbeb5204
2024-11-09T21:24:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/b3de371ab3eb4a66944c7863bbeb5204 as hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/b3de371ab3eb4a66944c7863bbeb5204
2024-11-09T21:24:51,162 INFO  [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/b3de371ab3eb4a66944c7863bbeb5204, entries=1, sequenceid=18, filesize=5.9 K
2024-11-09T21:24:51,163 INFO  [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7aadedad35e084e85fd0104228cb95a3 in 31ms, sequenceid=18, compaction requested=false
2024-11-09T21:24:51,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 7aadedad35e084e85fd0104228cb95a3:

2024-11-09T21:24:51,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:24:51,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-09T21:24:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-11-09T21:24:51,167 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-09T21:24:51,167 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 188 msec
2024-11-09T21:24:51,170 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 194 msec
2024-11-09T21:24:51,284 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:51,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:51,549 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:52,285 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:52,541 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:52,550 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:53,286 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:53,542 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:53,551 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:54,287 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:54,543 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:54,552 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:55,288 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:55,544 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:55,553 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:55,981 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 7aadedad35e084e85fd0104228cb95a3, had cached 0 bytes from a total of 14329
2024-11-09T21:24:56,289 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:56,546 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:56,554 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:57,290 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:57,547 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:57,555 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:58,291 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:58,548 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:58,556 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:59,292 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:59,549 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:24:59,557 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:00,293 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:00,550 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:00,557 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-09T21:25:01,005 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-09T21:25:01,013 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C37125%2C1731187449492.1731187501013
2024-11-09T21:25:01,023 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,023 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,024 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,024 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,024 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,024 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/WALs/d86ff69d6719,37125,1731187449492/d86ff69d6719%2C37125%2C1731187449492.1731187490966 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/WALs/d86ff69d6719,37125,1731187449492/d86ff69d6719%2C37125%2C1731187449492.1731187501013
2024-11-09T21:25:01,025 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38679:38679),(127.0.0.1/127.0.0.1:44959:44959)]
2024-11-09T21:25:01,025 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/WALs/d86ff69d6719,37125,1731187449492/d86ff69d6719%2C37125%2C1731187449492.1731187490966 is not closed yet, will try archiving it next time
2024-11-09T21:25:01,025 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-09T21:25:01,025 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/WALs/d86ff69d6719,37125,1731187449492/d86ff69d6719%2C37125%2C1731187449492.1731187480812 to hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/oldWALs/d86ff69d6719%2C37125%2C1731187449492.1731187480812
2024-11-09T21:25:01,025 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-09T21:25:01,025 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:25:01,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:01,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:01,026 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741842_1018 (size=2026)
2024-11-09T21:25:01,026 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-09T21:25:01,026 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=376633276, stopped=false
2024-11-09T21:25:01,026 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-09T21:25:01,026 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d86ff69d6719,37815,1731187449361
2024-11-09T21:25:01,027 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741842_1018 (size=2026)
2024-11-09T21:25:01,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-09T21:25:01,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-09T21:25:01,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:01,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:01,086 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-09T21:25:01,087 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-09T21:25:01,087 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:25:01,087 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:01,087 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:25:01,088 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd86ff69d6719,37125,1731187449492' *****
2024-11-09T21:25:01,088 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-09T21:25:01,088 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:25:01,088 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-09T21:25:01,089 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-09T21:25:01,089 INFO  [RS:0;d86ff69d6719:37125 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-09T21:25:01,089 INFO  [RS:0;d86ff69d6719:37125 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-09T21:25:01,089 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer(3091): Received CLOSE for 7aadedad35e084e85fd0104228cb95a3
2024-11-09T21:25:01,090 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer(959): stopping server d86ff69d6719,37125,1731187449492
2024-11-09T21:25:01,090 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-09T21:25:01,090 INFO  [RS:0;d86ff69d6719:37125 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d86ff69d6719:37125.
2024-11-09T21:25:01,090 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7aadedad35e084e85fd0104228cb95a3, disabling compactions & flushes
2024-11-09T21:25:01,090 DEBUG [RS:0;d86ff69d6719:37125 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:25:01,090 DEBUG [RS:0;d86ff69d6719:37125 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:01,090 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:25:01,090 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:25:01,090 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-09T21:25:01,090 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3. after waiting 0 ms
2024-11-09T21:25:01,090 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-09T21:25:01,090 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:25:01,090 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-09T21:25:01,090 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-09T21:25:01,090 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7aadedad35e084e85fd0104228cb95a3 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-09T21:25:01,091 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-09T21:25:01,091 DEBUG [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 7aadedad35e084e85fd0104228cb95a3=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.}
2024-11-09T21:25:01,091 DEBUG [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7aadedad35e084e85fd0104228cb95a3
2024-11-09T21:25:01,091 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-09T21:25:01,091 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-09T21:25:01,091 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-09T21:25:01,091 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-09T21:25:01,091 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-09T21:25:01,091 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-11-09T21:25:01,095 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/d319cc6eb1334288b9f4ddee3825d640 is 1080, key is row0001/info:/1731187501008/Put/seqid=0
2024-11-09T21:25:01,100 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741845_1021 (size=6033)
2024-11-09T21:25:01,100 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741845_1021 (size=6033)
2024-11-09T21:25:01,101 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/d319cc6eb1334288b9f4ddee3825d640
2024-11-09T21:25:01,108 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/.tmp/info/d319cc6eb1334288b9f4ddee3825d640 as hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/d319cc6eb1334288b9f4ddee3825d640
2024-11-09T21:25:01,111 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/.tmp/info/008f3878d1804d9099d65e31c14f2e9c is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3./info:regioninfo/1731187450997/Put/seqid=0
2024-11-09T21:25:01,114 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/d319cc6eb1334288b9f4ddee3825d640, entries=1, sequenceid=22, filesize=5.9 K
2024-11-09T21:25:01,115 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7aadedad35e084e85fd0104228cb95a3 in 25ms, sequenceid=22, compaction requested=true
2024-11-09T21:25:01,115 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741846_1022 (size=7308)
2024-11-09T21:25:01,115 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741846_1022 (size=7308)
2024-11-09T21:25:01,116 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/.tmp/info/008f3878d1804d9099d65e31c14f2e9c
2024-11-09T21:25:01,120 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/0ce318ce8f4940799e6cf1564bc3fd6d, hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/a0cf87848f984ac1acbefff355a3bd17, hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/f4acac1a9b494ee1a3100e42df13ee64] to archive
2024-11-09T21:25:01,121 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-09T21:25:01,123 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/0ce318ce8f4940799e6cf1564bc3fd6d to hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/0ce318ce8f4940799e6cf1564bc3fd6d
2024-11-09T21:25:01,124 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/a0cf87848f984ac1acbefff355a3bd17 to hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/a0cf87848f984ac1acbefff355a3bd17
2024-11-09T21:25:01,125 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/f4acac1a9b494ee1a3100e42df13ee64 to hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/info/f4acac1a9b494ee1a3100e42df13ee64
2024-11-09T21:25:01,126 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d86ff69d6719:37815 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-09T21:25:01,126 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [0ce318ce8f4940799e6cf1564bc3fd6d=6033, a0cf87848f984ac1acbefff355a3bd17=6033, f4acac1a9b494ee1a3100e42df13ee64=6033]
2024-11-09T21:25:01,129 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7aadedad35e084e85fd0104228cb95a3/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-11-09T21:25:01,129 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:25:01,130 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7aadedad35e084e85fd0104228cb95a3:
Waiting for close lock at 1731187501090Running coprocessor pre-close hooks at 1731187501090Disabling compacts and flushes for region at 1731187501090Disabling writes for close at 1731187501090Obtaining lock to block concurrent updates at 1731187501090Preparing flush snapshotting stores in 7aadedad35e084e85fd0104228cb95a3 at 1731187501090Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1731187501091 (+1 ms)Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3. at 1731187501092 (+1 ms)Flushing 7aadedad35e084e85fd0104228cb95a3/info: creating writer at 1731187501092Flushing 7aadedad35e084e85fd0104228cb95a3/info: appending metadata at 1731187501095 (+3 ms)Flushing 7aadedad35e084e85fd0104228cb95a3/info: closing flushed file at 1731187501095Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@634bca4e: reopening flushed file at 1731187501107 (+12 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7aadedad35e084e85fd0104228cb95a3 in 25ms, sequenceid=22, compaction requested=true at 1731187501115 (+8 ms)Writing region close event to WAL at 1731187501126 (+11 ms)Running coprocessor post-close hooks at 1731187501129 (+3 ms)Closed at 1731187501129
2024-11-09T21:25:01,130 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731187450632.7aadedad35e084e85fd0104228cb95a3.
2024-11-09T21:25:01,134 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/.tmp/ns/fb2332943e044503a86c0c9694a6c86f is 43, key is default/ns:d/1731187450534/Put/seqid=0
2024-11-09T21:25:01,138 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741847_1023 (size=5153)
2024-11-09T21:25:01,138 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741847_1023 (size=5153)
2024-11-09T21:25:01,138 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/.tmp/ns/fb2332943e044503a86c0c9694a6c86f
2024-11-09T21:25:01,158 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/.tmp/table/206c3ea757e4445e922703cb4eedd393 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1731187451008/Put/seqid=0
2024-11-09T21:25:01,162 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741848_1024 (size=5508)
2024-11-09T21:25:01,162 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741848_1024 (size=5508)
2024-11-09T21:25:01,163 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/.tmp/table/206c3ea757e4445e922703cb4eedd393
2024-11-09T21:25:01,168 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/.tmp/info/008f3878d1804d9099d65e31c14f2e9c as hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/info/008f3878d1804d9099d65e31c14f2e9c
2024-11-09T21:25:01,174 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/info/008f3878d1804d9099d65e31c14f2e9c, entries=10, sequenceid=11, filesize=7.1 K
2024-11-09T21:25:01,175 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/.tmp/ns/fb2332943e044503a86c0c9694a6c86f as hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/ns/fb2332943e044503a86c0c9694a6c86f
2024-11-09T21:25:01,180 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/ns/fb2332943e044503a86c0c9694a6c86f, entries=2, sequenceid=11, filesize=5.0 K
2024-11-09T21:25:01,181 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/.tmp/table/206c3ea757e4445e922703cb4eedd393 as hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/table/206c3ea757e4445e922703cb4eedd393
2024-11-09T21:25:01,186 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/table/206c3ea757e4445e922703cb4eedd393, entries=2, sequenceid=11, filesize=5.4 K
2024-11-09T21:25:01,187 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=11, compaction requested=false
2024-11-09T21:25:01,190 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-09T21:25:01,191 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-09T21:25:01,191 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-09T21:25:01,191 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731187501091Running coprocessor pre-close hooks at 1731187501091Disabling compacts and flushes for region at 1731187501091Disabling writes for close at 1731187501091Obtaining lock to block concurrent updates at 1731187501091Preparing flush snapshotting stores in 1588230740 at 1731187501091Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1731187501092 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731187501092Flushing 1588230740/info: creating writer at 1731187501092Flushing 1588230740/info: appending metadata at 1731187501110 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731187501110Flushing 1588230740/ns: creating writer at 1731187501120 (+10 ms)Flushing 1588230740/ns: appending metadata at 1731187501134 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731187501134Flushing 1588230740/table: creating writer at 1731187501144 (+10 ms)Flushing 1588230740/table: appending metadata at 1731187501157 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731187501157Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f5be1ed: reopening flushed file at 1731187501167 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fad63ec: reopening flushed file at 1731187501174 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e974914: reopening flushed file at 1731187501180 (+6 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=11, compaction requested=false at 1731187501187 (+7 ms)Writing region close event to WAL at 1731187501188 (+1 ms)Running coprocessor post-close hooks at 1731187501191 (+3 ms)Closed at 1731187501191
2024-11-09T21:25:01,191 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-09T21:25:01,291 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer(976): stopping server d86ff69d6719,37125,1731187449492; all regions closed.
2024-11-09T21:25:01,291 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,291 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,291 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,292 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,292 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,293 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741834_1010 (size=3306)
2024-11-09T21:25:01,293 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741834_1010 (size=3306)
2024-11-09T21:25:01,294 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:01,295 DEBUG [RS:0;d86ff69d6719:37125 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/oldWALs
2024-11-09T21:25:01,295 INFO  [RS:0;d86ff69d6719:37125 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d86ff69d6719%2C37125%2C1731187449492.meta:.meta(num 1731187450447)
2024-11-09T21:25:01,296 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,296 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,296 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,296 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,296 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,297 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741844_1020 (size=1252)
2024-11-09T21:25:01,297 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741844_1020 (size=1252)
2024-11-09T21:25:01,300 DEBUG [RS:0;d86ff69d6719:37125 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/oldWALs
2024-11-09T21:25:01,300 INFO  [RS:0;d86ff69d6719:37125 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d86ff69d6719%2C37125%2C1731187449492:(num 1731187501013)
2024-11-09T21:25:01,300 DEBUG [RS:0;d86ff69d6719:37125 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:01,300 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.LeaseManager(133): Closed leases
2024-11-09T21:25:01,301 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-09T21:25:01,301 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.ChoreService(370): Chore service for: regionserver/d86ff69d6719:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-09T21:25:01,301 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-09T21:25:01,301 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-09T21:25:01,301 INFO  [RS:0;d86ff69d6719:37125 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37125
2024-11-09T21:25:01,333 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-09T21:25:01,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d86ff69d6719,37125,1731187449492
2024-11-09T21:25:01,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T21:25:01,344 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d86ff69d6719,37125,1731187449492]
2024-11-09T21:25:01,352 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d86ff69d6719,37125,1731187449492 already deleted, retry=false
2024-11-09T21:25:01,352 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d86ff69d6719,37125,1731187449492 expired; onlineServers=0
2024-11-09T21:25:01,352 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd86ff69d6719,37815,1731187449361' *****
2024-11-09T21:25:01,352 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-09T21:25:01,353 INFO  [M:0;d86ff69d6719:37815 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-09T21:25:01,353 INFO  [M:0;d86ff69d6719:37815 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-09T21:25:01,353 DEBUG [M:0;d86ff69d6719:37815 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-09T21:25:01,353 DEBUG [M:0;d86ff69d6719:37815 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-09T21:25:01,353 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-09T21:25:01,353 DEBUG [master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187449786 {}] cleaner.HFileCleaner(306): Exit Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187449786,5,FailOnTimeoutGroup]
2024-11-09T21:25:01,353 DEBUG [master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187449786 {}] cleaner.HFileCleaner(306): Exit Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187449786,5,FailOnTimeoutGroup]
2024-11-09T21:25:01,354 INFO  [M:0;d86ff69d6719:37815 {}] hbase.ChoreService(370): Chore service for: master/d86ff69d6719:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-09T21:25:01,354 INFO  [M:0;d86ff69d6719:37815 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-09T21:25:01,354 DEBUG [M:0;d86ff69d6719:37815 {}] master.HMaster(1795): Stopping service threads
2024-11-09T21:25:01,355 INFO  [M:0;d86ff69d6719:37815 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-09T21:25:01,355 INFO  [M:0;d86ff69d6719:37815 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-09T21:25:01,355 INFO  [M:0;d86ff69d6719:37815 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-09T21:25:01,356 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-09T21:25:01,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-09T21:25:01,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:01,366 DEBUG [M:0;d86ff69d6719:37815 {}] zookeeper.ZKUtil(347): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-09T21:25:01,366 WARN  [M:0;d86ff69d6719:37815 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-09T21:25:01,367 INFO  [M:0;d86ff69d6719:37815 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/.lastflushedseqids
2024-11-09T21:25:01,372 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741849_1025 (size=130)
2024-11-09T21:25:01,373 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741849_1025 (size=130)
2024-11-09T21:25:01,373 INFO  [M:0;d86ff69d6719:37815 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-09T21:25:01,373 INFO  [M:0;d86ff69d6719:37815 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-09T21:25:01,373 DEBUG [M:0;d86ff69d6719:37815 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-09T21:25:01,373 INFO  [M:0;d86ff69d6719:37815 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:01,373 DEBUG [M:0;d86ff69d6719:37815 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:01,373 DEBUG [M:0;d86ff69d6719:37815 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-09T21:25:01,374 DEBUG [M:0;d86ff69d6719:37815 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:01,374 INFO  [M:0;d86ff69d6719:37815 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.60 KB heapSize=55.01 KB
2024-11-09T21:25:01,393 DEBUG [M:0;d86ff69d6719:37815 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f7cf2d173aa140f4be701f0b6c064f3b is 82, key is hbase:meta,,1/info:regioninfo/1731187450479/Put/seqid=0
2024-11-09T21:25:01,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741850_1026 (size=5672)
2024-11-09T21:25:01,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741850_1026 (size=5672)
2024-11-09T21:25:01,398 INFO  [M:0;d86ff69d6719:37815 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f7cf2d173aa140f4be701f0b6c064f3b
2024-11-09T21:25:01,418 DEBUG [M:0;d86ff69d6719:37815 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9f537ba05d0b40bdb9036c74d894c6d0 is 799, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731187451013/Put/seqid=0
2024-11-09T21:25:01,422 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741851_1027 (size=7824)
2024-11-09T21:25:01,422 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741851_1027 (size=7824)
2024-11-09T21:25:01,423 INFO  [M:0;d86ff69d6719:37815 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.00 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9f537ba05d0b40bdb9036c74d894c6d0
2024-11-09T21:25:01,427 INFO  [M:0;d86ff69d6719:37815 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9f537ba05d0b40bdb9036c74d894c6d0
2024-11-09T21:25:01,443 DEBUG [M:0;d86ff69d6719:37815 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bb6f7cecb44641c1a72d0ec40238d94e is 69, key is d86ff69d6719,37125,1731187449492/rs:state/1731187449850/Put/seqid=0
2024-11-09T21:25:01,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:25:01,444 INFO  [RS:0;d86ff69d6719:37125 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-09T21:25:01,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37125-0x10122c5b0c20001, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:25:01,444 INFO  [RS:0;d86ff69d6719:37125 {}] regionserver.HRegionServer(1031): Exiting; stopping=d86ff69d6719,37125,1731187449492; zookeeper connection closed.
2024-11-09T21:25:01,445 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6a7f9427 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6a7f9427
2024-11-09T21:25:01,445 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-09T21:25:01,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741852_1028 (size=5156)
2024-11-09T21:25:01,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741852_1028 (size=5156)
2024-11-09T21:25:01,448 INFO  [M:0;d86ff69d6719:37815 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bb6f7cecb44641c1a72d0ec40238d94e
2024-11-09T21:25:01,466 DEBUG [M:0;d86ff69d6719:37815 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a099174772394a45bb0e30223d221291 is 52, key is load_balancer_on/state:d/1731187450628/Put/seqid=0
2024-11-09T21:25:01,471 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741853_1029 (size=5056)
2024-11-09T21:25:01,471 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741853_1029 (size=5056)
2024-11-09T21:25:01,471 INFO  [M:0;d86ff69d6719:37815 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a099174772394a45bb0e30223d221291
2024-11-09T21:25:01,476 DEBUG [M:0;d86ff69d6719:37815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f7cf2d173aa140f4be701f0b6c064f3b as hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f7cf2d173aa140f4be701f0b6c064f3b
2024-11-09T21:25:01,482 INFO  [M:0;d86ff69d6719:37815 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f7cf2d173aa140f4be701f0b6c064f3b, entries=8, sequenceid=121, filesize=5.5 K
2024-11-09T21:25:01,483 DEBUG [M:0;d86ff69d6719:37815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9f537ba05d0b40bdb9036c74d894c6d0 as hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9f537ba05d0b40bdb9036c74d894c6d0
2024-11-09T21:25:01,489 INFO  [M:0;d86ff69d6719:37815 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9f537ba05d0b40bdb9036c74d894c6d0
2024-11-09T21:25:01,489 INFO  [M:0;d86ff69d6719:37815 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9f537ba05d0b40bdb9036c74d894c6d0, entries=14, sequenceid=121, filesize=7.6 K
2024-11-09T21:25:01,490 DEBUG [M:0;d86ff69d6719:37815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bb6f7cecb44641c1a72d0ec40238d94e as hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bb6f7cecb44641c1a72d0ec40238d94e
2024-11-09T21:25:01,494 INFO  [M:0;d86ff69d6719:37815 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bb6f7cecb44641c1a72d0ec40238d94e, entries=1, sequenceid=121, filesize=5.0 K
2024-11-09T21:25:01,495 DEBUG [M:0;d86ff69d6719:37815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a099174772394a45bb0e30223d221291 as hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a099174772394a45bb0e30223d221291
2024-11-09T21:25:01,500 INFO  [M:0;d86ff69d6719:37815 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35747/user/jenkins/test-data/b7c19f0a-42b6-89be-1062-c5045a2c5023/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a099174772394a45bb0e30223d221291, entries=1, sequenceid=121, filesize=4.9 K
2024-11-09T21:25:01,501 INFO  [M:0;d86ff69d6719:37815 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.60 KB/44650, heapSize ~54.95 KB/56264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 127ms, sequenceid=121, compaction requested=false
2024-11-09T21:25:01,502 INFO  [M:0;d86ff69d6719:37815 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:01,503 DEBUG [M:0;d86ff69d6719:37815 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731187501373Disabling compacts and flushes for region at 1731187501373Disabling writes for close at 1731187501374 (+1 ms)Obtaining lock to block concurrent updates at 1731187501374Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731187501374Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44650, getHeapSize=56264, getOffHeapSize=0, getCellsCount=140 at 1731187501374Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731187501375 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731187501375Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731187501393 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731187501393Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731187501402 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731187501417 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731187501417Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731187501427 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731187501442 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731187501442Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731187501452 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731187501466 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731187501466Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@590a7c8c: reopening flushed file at 1731187501475 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15ebbfaa: reopening flushed file at 1731187501482 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@776f90a4: reopening flushed file at 1731187501489 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e3d6caf: reopening flushed file at 1731187501495 (+6 ms)Finished flush of dataSize ~43.60 KB/44650, heapSize ~54.95 KB/56264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 127ms, sequenceid=121, compaction requested=false at 1731187501501 (+6 ms)Writing region close event to WAL at 1731187501502 (+1 ms)Closed at 1731187501502
2024-11-09T21:25:01,503 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,503 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,503 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,503 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,503 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:01,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44033 is added to blk_1073741830_1006 (size=53047)
2024-11-09T21:25:01,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45733 is added to blk_1073741830_1006 (size=53047)
2024-11-09T21:25:01,506 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-09T21:25:01,506 INFO  [M:0;d86ff69d6719:37815 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-09T21:25:01,506 INFO  [M:0;d86ff69d6719:37815 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37815
2024-11-09T21:25:01,506 INFO  [M:0;d86ff69d6719:37815 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-09T21:25:01,551 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:01,558 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:01,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:25:01,633 INFO  [M:0;d86ff69d6719:37815 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-09T21:25:01,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37815-0x10122c5b0c20000, quorum=127.0.0.1:53181, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:25:01,636 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3aa8039f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:25:01,637 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@55244549{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:25:01,637 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:25:01,637 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62aa92bd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:25:01,638 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e428b73{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/hadoop.log.dir/,STOPPED}
2024-11-09T21:25:01,641 WARN  [BP-1494917969-172.17.0.2-1731187447741 heartbeating to localhost/127.0.0.1:35747 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:25:01,641 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:25:01,641 WARN  [BP-1494917969-172.17.0.2-1731187447741 heartbeating to localhost/127.0.0.1:35747 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1494917969-172.17.0.2-1731187447741 (Datanode Uuid 9926d156-8d06-4340-b0a5-b69cdaf2243e) service to localhost/127.0.0.1:35747
2024-11-09T21:25:01,641 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:25:01,642 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/cluster_d0e49250-96c8-093e-ed32-ea6c84048991/data/data3/current/BP-1494917969-172.17.0.2-1731187447741 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:25:01,642 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/cluster_d0e49250-96c8-093e-ed32-ea6c84048991/data/data4/current/BP-1494917969-172.17.0.2-1731187447741 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:25:01,642 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:25:01,645 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6719db8a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:25:01,645 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3e2a30ba{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:25:01,646 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:25:01,646 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6f17b515{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:25:01,646 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@280e4ad5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/hadoop.log.dir/,STOPPED}
2024-11-09T21:25:01,647 WARN  [BP-1494917969-172.17.0.2-1731187447741 heartbeating to localhost/127.0.0.1:35747 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:25:01,647 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:25:01,647 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:25:01,647 WARN  [BP-1494917969-172.17.0.2-1731187447741 heartbeating to localhost/127.0.0.1:35747 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1494917969-172.17.0.2-1731187447741 (Datanode Uuid 5c5b9681-d557-4357-ad59-c91cc8dc8620) service to localhost/127.0.0.1:35747
2024-11-09T21:25:01,648 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/cluster_d0e49250-96c8-093e-ed32-ea6c84048991/data/data1/current/BP-1494917969-172.17.0.2-1731187447741 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:25:01,649 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/cluster_d0e49250-96c8-093e-ed32-ea6c84048991/data/data2/current/BP-1494917969-172.17.0.2-1731187447741 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:25:01,649 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:25:01,656 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4e873b68{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-09T21:25:01,656 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22fd1e24{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:25:01,656 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:25:01,656 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6bd1231f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:25:01,656 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8afa355{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/hadoop.log.dir/,STOPPED}
2024-11-09T21:25:01,664 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-09T21:25:01,686 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-09T21:25:01,693 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=209 (was 183)
Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:35747
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:35747
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/d86ff69d6719:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35747 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35747
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35747
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35747
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35747 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35747 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=22 (was 44), ProcessCount=11 (was 11), AvailableMemoryMB=17993 (was 18062)
2024-11-09T21:25:01,700 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=209, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=22, ProcessCount=11, AvailableMemoryMB=17992
2024-11-09T21:25:01,700 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-09T21:25:01,700 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/hadoop.log.dir so I do NOT create it in target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f
2024-11-09T21:25:01,700 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f6701b34-91f3-916c-13e2-a69ff91bf928/hadoop.tmp.dir so I do NOT create it in target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f
2024-11-09T21:25:01,700 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/cluster_e839f11b-8a2a-e3b6-d5fe-08fab413cc82, deleteOnExit=true
2024-11-09T21:25:01,700 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-09T21:25:01,700 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/test.cache.data in system properties and HBase conf
2024-11-09T21:25:01,700 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/hadoop.tmp.dir in system properties and HBase conf
2024-11-09T21:25:01,700 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/hadoop.log.dir in system properties and HBase conf
2024-11-09T21:25:01,700 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-09T21:25:01,700 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-09T21:25:01,700 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-09T21:25:01,701 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-09T21:25:01,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-09T21:25:01,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-09T21:25:01,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-09T21:25:01,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-09T21:25:01,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-09T21:25:01,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-09T21:25:01,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-09T21:25:01,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-09T21:25:01,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-09T21:25:01,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/nfs.dump.dir in system properties and HBase conf
2024-11-09T21:25:01,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/java.io.tmpdir in system properties and HBase conf
2024-11-09T21:25:01,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-09T21:25:01,702 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-09T21:25:01,702 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-09T21:25:01,714 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-09T21:25:01,867 INFO  [regionserver/d86ff69d6719:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-09T21:25:01,948 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:25:01,951 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:25:01,952 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:25:01,953 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:25:01,953 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-09T21:25:01,953 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:25:01,954 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b28e221{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:25:01,954 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@64d2170c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:25:02,046 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@59fd7f75{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/java.io.tmpdir/jetty-localhost-35273-hadoop-hdfs-3_4_1-tests_jar-_-any-8760330792435491939/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-09T21:25:02,046 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4b9d1fb8{HTTP/1.1, (http/1.1)}{localhost:35273}
2024-11-09T21:25:02,046 INFO  [Time-limited test {}] server.Server(415): Started @249465ms
2024-11-09T21:25:02,057 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-09T21:25:02,265 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:25:02,268 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:25:02,268 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:25:02,268 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:25:02,268 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-09T21:25:02,269 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49490ce4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:25:02,269 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@19fe8881{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:25:02,294 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:02,361 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@47bcda8c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/java.io.tmpdir/jetty-localhost-34749-hadoop-hdfs-3_4_1-tests_jar-_-any-16693720836555857918/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:25:02,361 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@29765213{HTTP/1.1, (http/1.1)}{localhost:34749}
2024-11-09T21:25:02,362 INFO  [Time-limited test {}] server.Server(415): Started @249780ms
2024-11-09T21:25:02,362 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:25:02,387 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:25:02,391 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:25:02,392 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:25:02,392 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:25:02,392 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-09T21:25:02,392 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48123521{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:25:02,393 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3c5fec98{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:25:02,483 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@56931997{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/java.io.tmpdir/jetty-localhost-34369-hadoop-hdfs-3_4_1-tests_jar-_-any-14630542372487611195/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:25:02,484 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@19c2a7a5{HTTP/1.1, (http/1.1)}{localhost:34369}
2024-11-09T21:25:02,484 INFO  [Time-limited test {}] server.Server(415): Started @249903ms
2024-11-09T21:25:02,485 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:25:02,551 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:02,559 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:03,132 WARN  [Thread-1957 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/cluster_e839f11b-8a2a-e3b6-d5fe-08fab413cc82/data/data1/current/BP-1597026677-172.17.0.2-1731187501718/current, will proceed with Du for space computation calculation, 
2024-11-09T21:25:03,132 WARN  [Thread-1958 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/cluster_e839f11b-8a2a-e3b6-d5fe-08fab413cc82/data/data2/current/BP-1597026677-172.17.0.2-1731187501718/current, will proceed with Du for space computation calculation, 
2024-11-09T21:25:03,152 WARN  [Thread-1921 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:25:03,154 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x20a81705f177c69d with lease ID 0xb250f5b070d9544: Processing first storage report for DS-fb49ae11-1596-462c-827e-7512395bc036 from datanode DatanodeRegistration(127.0.0.1:38675, datanodeUuid=fe225878-ad5d-47ea-bc05-6f4eede53cc5, infoPort=42767, infoSecurePort=0, ipcPort=33417, storageInfo=lv=-57;cid=testClusterID;nsid=1989005784;c=1731187501718)
2024-11-09T21:25:03,154 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20a81705f177c69d with lease ID 0xb250f5b070d9544: from storage DS-fb49ae11-1596-462c-827e-7512395bc036 node DatanodeRegistration(127.0.0.1:38675, datanodeUuid=fe225878-ad5d-47ea-bc05-6f4eede53cc5, infoPort=42767, infoSecurePort=0, ipcPort=33417, storageInfo=lv=-57;cid=testClusterID;nsid=1989005784;c=1731187501718), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:25:03,154 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x20a81705f177c69d with lease ID 0xb250f5b070d9544: Processing first storage report for DS-81fd5213-e006-42b8-93ee-2db8550a1939 from datanode DatanodeRegistration(127.0.0.1:38675, datanodeUuid=fe225878-ad5d-47ea-bc05-6f4eede53cc5, infoPort=42767, infoSecurePort=0, ipcPort=33417, storageInfo=lv=-57;cid=testClusterID;nsid=1989005784;c=1731187501718)
2024-11-09T21:25:03,154 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20a81705f177c69d with lease ID 0xb250f5b070d9544: from storage DS-81fd5213-e006-42b8-93ee-2db8550a1939 node DatanodeRegistration(127.0.0.1:38675, datanodeUuid=fe225878-ad5d-47ea-bc05-6f4eede53cc5, infoPort=42767, infoSecurePort=0, ipcPort=33417, storageInfo=lv=-57;cid=testClusterID;nsid=1989005784;c=1731187501718), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:25:03,295 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:03,299 WARN  [Thread-1968 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/cluster_e839f11b-8a2a-e3b6-d5fe-08fab413cc82/data/data3/current/BP-1597026677-172.17.0.2-1731187501718/current, will proceed with Du for space computation calculation, 
2024-11-09T21:25:03,299 WARN  [Thread-1969 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/cluster_e839f11b-8a2a-e3b6-d5fe-08fab413cc82/data/data4/current/BP-1597026677-172.17.0.2-1731187501718/current, will proceed with Du for space computation calculation, 
2024-11-09T21:25:03,319 WARN  [Thread-1944 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:25:03,321 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x534d907410bab8b0 with lease ID 0xb250f5b070d9545: Processing first storage report for DS-ee63b0cf-f36e-4fba-b720-8ee826b1fecb from datanode DatanodeRegistration(127.0.0.1:36959, datanodeUuid=7ad3e5e1-0d11-403e-9383-4d51fdcbf210, infoPort=43417, infoSecurePort=0, ipcPort=35007, storageInfo=lv=-57;cid=testClusterID;nsid=1989005784;c=1731187501718)
2024-11-09T21:25:03,321 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x534d907410bab8b0 with lease ID 0xb250f5b070d9545: from storage DS-ee63b0cf-f36e-4fba-b720-8ee826b1fecb node DatanodeRegistration(127.0.0.1:36959, datanodeUuid=7ad3e5e1-0d11-403e-9383-4d51fdcbf210, infoPort=43417, infoSecurePort=0, ipcPort=35007, storageInfo=lv=-57;cid=testClusterID;nsid=1989005784;c=1731187501718), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:25:03,321 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x534d907410bab8b0 with lease ID 0xb250f5b070d9545: Processing first storage report for DS-095e2940-e631-4066-a54c-5e6cf4066ea0 from datanode DatanodeRegistration(127.0.0.1:36959, datanodeUuid=7ad3e5e1-0d11-403e-9383-4d51fdcbf210, infoPort=43417, infoSecurePort=0, ipcPort=35007, storageInfo=lv=-57;cid=testClusterID;nsid=1989005784;c=1731187501718)
2024-11-09T21:25:03,321 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x534d907410bab8b0 with lease ID 0xb250f5b070d9545: from storage DS-095e2940-e631-4066-a54c-5e6cf4066ea0 node DatanodeRegistration(127.0.0.1:36959, datanodeUuid=7ad3e5e1-0d11-403e-9383-4d51fdcbf210, infoPort=43417, infoSecurePort=0, ipcPort=35007, storageInfo=lv=-57;cid=testClusterID;nsid=1989005784;c=1731187501718), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:25:03,410 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f
2024-11-09T21:25:03,412 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/cluster_e839f11b-8a2a-e3b6-d5fe-08fab413cc82/zookeeper_0, clientPort=57808, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/cluster_e839f11b-8a2a-e3b6-d5fe-08fab413cc82/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/cluster_e839f11b-8a2a-e3b6-d5fe-08fab413cc82/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-09T21:25:03,413 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57808
2024-11-09T21:25:03,413 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:25:03,414 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:25:03,423 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741825_1001 (size=7)
2024-11-09T21:25:03,423 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741825_1001 (size=7)
2024-11-09T21:25:03,424 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11 with version=8
2024-11-09T21:25:03,424 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/hbase-staging
2024-11-09T21:25:03,426 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d86ff69d6719:0 server-side Connection retries=45
2024-11-09T21:25:03,426 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:25:03,426 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T21:25:03,426 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T21:25:03,426 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:25:03,427 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T21:25:03,427 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-09T21:25:03,427 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T21:25:03,427 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46193
2024-11-09T21:25:03,428 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46193 connecting to ZooKeeper ensemble=127.0.0.1:57808
2024-11-09T21:25:03,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:461930x0, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T21:25:03,478 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46193-0x10122c683f90000 connected
2024-11-09T21:25:03,552 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:03,559 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:03,561 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:25:03,564 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:25:03,568 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:25:03,569 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11, hbase.cluster.distributed=false
2024-11-09T21:25:03,570 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T21:25:03,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46193
2024-11-09T21:25:03,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46193
2024-11-09T21:25:03,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46193
2024-11-09T21:25:03,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46193
2024-11-09T21:25:03,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46193
2024-11-09T21:25:03,586 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d86ff69d6719:0 server-side Connection retries=45
2024-11-09T21:25:03,586 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:25:03,586 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T21:25:03,586 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T21:25:03,586 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:25:03,586 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T21:25:03,586 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-09T21:25:03,586 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T21:25:03,587 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34269
2024-11-09T21:25:03,588 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34269 connecting to ZooKeeper ensemble=127.0.0.1:57808
2024-11-09T21:25:03,589 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:25:03,590 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:25:03,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:342690x0, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T21:25:03,602 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34269-0x10122c683f90001 connected
2024-11-09T21:25:03,602 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:25:03,602 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-09T21:25:03,603 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-09T21:25:03,604 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-09T21:25:03,605 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T21:25:03,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34269
2024-11-09T21:25:03,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34269
2024-11-09T21:25:03,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34269
2024-11-09T21:25:03,607 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34269
2024-11-09T21:25:03,607 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34269
2024-11-09T21:25:03,627 DEBUG [M:0;d86ff69d6719:46193 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d86ff69d6719:46193
2024-11-09T21:25:03,627 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d86ff69d6719,46193,1731187503426
2024-11-09T21:25:03,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:25:03,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:25:03,635 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d86ff69d6719,46193,1731187503426
2024-11-09T21:25:03,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-09T21:25:03,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:03,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:03,644 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-09T21:25:03,644 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d86ff69d6719,46193,1731187503426 from backup master directory
2024-11-09T21:25:03,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:25:03,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d86ff69d6719,46193,1731187503426
2024-11-09T21:25:03,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:25:03,651 WARN  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T21:25:03,651 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d86ff69d6719,46193,1731187503426
2024-11-09T21:25:03,655 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/hbase.id] with ID: 7e091e1c-2fe4-4522-9a15-da6501bf3fb0
2024-11-09T21:25:03,655 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/.tmp/hbase.id
2024-11-09T21:25:03,660 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741826_1002 (size=42)
2024-11-09T21:25:03,660 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741826_1002 (size=42)
2024-11-09T21:25:03,661 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/.tmp/hbase.id]:[hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/hbase.id]
2024-11-09T21:25:03,672 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:25:03,672 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-09T21:25:03,674 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-09T21:25:03,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:03,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:03,691 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741827_1003 (size=196)
2024-11-09T21:25:03,691 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741827_1003 (size=196)
2024-11-09T21:25:03,691 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-09T21:25:03,692 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-09T21:25:03,692 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:25:03,698 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741828_1004 (size=1189)
2024-11-09T21:25:03,698 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741828_1004 (size=1189)
2024-11-09T21:25:03,699 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store
2024-11-09T21:25:03,704 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741829_1005 (size=34)
2024-11-09T21:25:03,704 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741829_1005 (size=34)
2024-11-09T21:25:03,705 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:25:03,705 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-09T21:25:03,705 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:03,705 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:03,705 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-09T21:25:03,705 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:03,705 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:03,705 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731187503705Disabling compacts and flushes for region at 1731187503705Disabling writes for close at 1731187503705Writing region close event to WAL at 1731187503705Closed at 1731187503705
2024-11-09T21:25:03,706 WARN  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/.initializing
2024-11-09T21:25:03,706 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/WALs/d86ff69d6719,46193,1731187503426
2024-11-09T21:25:03,708 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C46193%2C1731187503426, suffix=, logDir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/WALs/d86ff69d6719,46193,1731187503426, archiveDir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/oldWALs, maxLogs=10
2024-11-09T21:25:03,708 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C46193%2C1731187503426.1731187503708
2024-11-09T21:25:03,712 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/WALs/d86ff69d6719,46193,1731187503426/d86ff69d6719%2C46193%2C1731187503426.1731187503708
2024-11-09T21:25:03,713 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42767:42767),(127.0.0.1/127.0.0.1:43417:43417)]
2024-11-09T21:25:03,713 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:25:03,714 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:25:03,714 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:25:03,714 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:25:03,719 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:25:03,720 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-09T21:25:03,720 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:03,720 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:03,720 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:25:03,721 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-09T21:25:03,721 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:03,721 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:25:03,721 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:25:03,722 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-09T21:25:03,722 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:03,723 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:25:03,723 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:25:03,723 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-09T21:25:03,724 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:03,724 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:25:03,724 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:25:03,725 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:25:03,725 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:25:03,726 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:25:03,726 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:25:03,726 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-09T21:25:03,727 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:25:03,729 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:25:03,730 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=763496, jitterRate=-0.02916574478149414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-09T21:25:03,730 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731187503714Initializing all the Stores at 1731187503714Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187503714Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187503718 (+4 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187503718Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187503718Cleaning up temporary data from old regions at 1731187503726 (+8 ms)Region opened successfully at 1731187503730 (+4 ms)
2024-11-09T21:25:03,731 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-09T21:25:03,734 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d8f963b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0
2024-11-09T21:25:03,735 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-09T21:25:03,735 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-09T21:25:03,735 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-09T21:25:03,735 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-09T21:25:03,735 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-09T21:25:03,736 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-09T21:25:03,736 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-09T21:25:03,740 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-09T21:25:03,741 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-09T21:25:03,760 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-09T21:25:03,760 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-09T21:25:03,761 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-09T21:25:03,768 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-09T21:25:03,768 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-09T21:25:03,769 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-09T21:25:03,776 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-09T21:25:03,777 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-09T21:25:03,785 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-09T21:25:03,787 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-09T21:25:03,793 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-09T21:25:03,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-09T21:25:03,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-09T21:25:03,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:03,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:03,802 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d86ff69d6719,46193,1731187503426, sessionid=0x10122c683f90000, setting cluster-up flag (Was=false)
2024-11-09T21:25:03,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:03,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:03,843 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-09T21:25:03,844 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d86ff69d6719,46193,1731187503426
2024-11-09T21:25:03,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:03,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:03,885 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-09T21:25:03,886 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d86ff69d6719,46193,1731187503426
2024-11-09T21:25:03,888 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-09T21:25:03,890 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-09T21:25:03,890 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-09T21:25:03,890 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-09T21:25:03,891 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d86ff69d6719,46193,1731187503426
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-09T21:25:03,893 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:25:03,893 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:25:03,893 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:25:03,893 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:25:03,893 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d86ff69d6719:0, corePoolSize=10, maxPoolSize=10
2024-11-09T21:25:03,894 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:03,894 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2
2024-11-09T21:25:03,894 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:03,895 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731187533895
2024-11-09T21:25:03,895 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-09T21:25:03,895 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-09T21:25:03,895 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-09T21:25:03,895 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-09T21:25:03,896 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-09T21:25:03,896 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-09T21:25:03,896 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:03,896 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-09T21:25:03,896 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:25:03,896 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-09T21:25:03,896 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-09T21:25:03,896 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-09T21:25:03,896 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-09T21:25:03,896 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-09T21:25:03,897 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187503897,5,FailOnTimeoutGroup]
2024-11-09T21:25:03,897 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187503897,5,FailOnTimeoutGroup]
2024-11-09T21:25:03,897 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:03,897 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-09T21:25:03,897 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:03,897 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:03,897 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:03,897 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-09T21:25:03,902 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741831_1007 (size=1321)
2024-11-09T21:25:03,903 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741831_1007 (size=1321)
2024-11-09T21:25:03,903 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-09T21:25:03,904 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11
2024-11-09T21:25:03,909 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(746): ClusterId : 7e091e1c-2fe4-4522-9a15-da6501bf3fb0
2024-11-09T21:25:03,909 DEBUG [RS:0;d86ff69d6719:34269 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-09T21:25:03,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741832_1008 (size=32)
2024-11-09T21:25:03,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741832_1008 (size=32)
2024-11-09T21:25:03,911 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:25:03,912 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-09T21:25:03,914 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-09T21:25:03,914 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:03,915 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:03,915 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-09T21:25:03,916 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-09T21:25:03,916 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:03,917 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:03,917 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-09T21:25:03,918 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-09T21:25:03,918 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:03,919 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:03,919 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-09T21:25:03,919 DEBUG [RS:0;d86ff69d6719:34269 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-09T21:25:03,919 DEBUG [RS:0;d86ff69d6719:34269 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-09T21:25:03,920 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-09T21:25:03,920 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:03,921 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:03,921 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-09T21:25:03,921 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740
2024-11-09T21:25:03,922 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740
2024-11-09T21:25:03,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-09T21:25:03,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-09T21:25:03,923 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-09T21:25:03,925 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-09T21:25:03,927 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:25:03,927 DEBUG [RS:0;d86ff69d6719:34269 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-09T21:25:03,927 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=793379, jitterRate=0.008833765983581543}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-09T21:25:03,927 DEBUG [RS:0;d86ff69d6719:34269 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@733b4e82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0
2024-11-09T21:25:03,928 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731187503911Initializing all the Stores at 1731187503912 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187503912Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187503912Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187503912Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187503912Cleaning up temporary data from old regions at 1731187503923 (+11 ms)Region opened successfully at 1731187503928 (+5 ms)
2024-11-09T21:25:03,928 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-09T21:25:03,928 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-09T21:25:03,928 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-09T21:25:03,928 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-09T21:25:03,928 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-09T21:25:03,928 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-09T21:25:03,929 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731187503928Disabling compacts and flushes for region at 1731187503928Disabling writes for close at 1731187503928Writing region close event to WAL at 1731187503928Closed at 1731187503928
2024-11-09T21:25:03,930 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:25:03,930 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-09T21:25:03,930 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-09T21:25:03,932 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-09T21:25:03,933 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-09T21:25:03,942 DEBUG [RS:0;d86ff69d6719:34269 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d86ff69d6719:34269
2024-11-09T21:25:03,942 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-09T21:25:03,942 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-09T21:25:03,942 DEBUG [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-09T21:25:03,943 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(2659): reportForDuty to master=d86ff69d6719,46193,1731187503426 with port=34269, startcode=1731187503585
2024-11-09T21:25:03,943 DEBUG [RS:0;d86ff69d6719:34269 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-09T21:25:03,945 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36223, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-11-09T21:25:03,945 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46193 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d86ff69d6719,34269,1731187503585
2024-11-09T21:25:03,945 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46193 {}] master.ServerManager(517): Registering regionserver=d86ff69d6719,34269,1731187503585
2024-11-09T21:25:03,947 DEBUG [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11
2024-11-09T21:25:03,947 DEBUG [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38249
2024-11-09T21:25:03,947 DEBUG [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-09T21:25:03,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T21:25:03,957 DEBUG [RS:0;d86ff69d6719:34269 {}] zookeeper.ZKUtil(111): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d86ff69d6719,34269,1731187503585
2024-11-09T21:25:03,957 WARN  [RS:0;d86ff69d6719:34269 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T21:25:03,958 INFO  [RS:0;d86ff69d6719:34269 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:25:03,958 DEBUG [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/WALs/d86ff69d6719,34269,1731187503585
2024-11-09T21:25:03,958 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d86ff69d6719,34269,1731187503585]
2024-11-09T21:25:03,961 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-09T21:25:03,962 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-09T21:25:03,963 INFO  [RS:0;d86ff69d6719:34269 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-09T21:25:03,963 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:03,963 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-09T21:25:03,964 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-09T21:25:03,964 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:03,964 DEBUG [RS:0;d86ff69d6719:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:03,964 DEBUG [RS:0;d86ff69d6719:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:03,964 DEBUG [RS:0;d86ff69d6719:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:03,964 DEBUG [RS:0;d86ff69d6719:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:03,964 DEBUG [RS:0;d86ff69d6719:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:03,964 DEBUG [RS:0;d86ff69d6719:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2
2024-11-09T21:25:03,964 DEBUG [RS:0;d86ff69d6719:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:03,964 DEBUG [RS:0;d86ff69d6719:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:03,964 DEBUG [RS:0;d86ff69d6719:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:03,964 DEBUG [RS:0;d86ff69d6719:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:03,964 DEBUG [RS:0;d86ff69d6719:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:03,965 DEBUG [RS:0;d86ff69d6719:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:03,965 DEBUG [RS:0;d86ff69d6719:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3
2024-11-09T21:25:03,965 DEBUG [RS:0;d86ff69d6719:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3
2024-11-09T21:25:03,965 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:03,965 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:03,965 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:03,965 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:03,965 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:03,965 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,34269,1731187503585-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T21:25:03,979 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-09T21:25:03,979 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,34269,1731187503585-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:03,979 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:03,979 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.Replication(171): d86ff69d6719,34269,1731187503585 started
2024-11-09T21:25:03,991 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:03,991 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(1482): Serving as d86ff69d6719,34269,1731187503585, RpcServer on d86ff69d6719/172.17.0.2:34269, sessionid=0x10122c683f90001
2024-11-09T21:25:03,991 DEBUG [RS:0;d86ff69d6719:34269 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-09T21:25:03,991 DEBUG [RS:0;d86ff69d6719:34269 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d86ff69d6719,34269,1731187503585
2024-11-09T21:25:03,991 DEBUG [RS:0;d86ff69d6719:34269 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,34269,1731187503585'
2024-11-09T21:25:03,991 DEBUG [RS:0;d86ff69d6719:34269 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-09T21:25:03,992 DEBUG [RS:0;d86ff69d6719:34269 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-09T21:25:03,992 DEBUG [RS:0;d86ff69d6719:34269 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-09T21:25:03,992 DEBUG [RS:0;d86ff69d6719:34269 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-09T21:25:03,992 DEBUG [RS:0;d86ff69d6719:34269 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d86ff69d6719,34269,1731187503585
2024-11-09T21:25:03,993 DEBUG [RS:0;d86ff69d6719:34269 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,34269,1731187503585'
2024-11-09T21:25:03,993 DEBUG [RS:0;d86ff69d6719:34269 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-09T21:25:03,993 DEBUG [RS:0;d86ff69d6719:34269 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-09T21:25:03,993 DEBUG [RS:0;d86ff69d6719:34269 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-09T21:25:03,993 INFO  [RS:0;d86ff69d6719:34269 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-09T21:25:03,993 INFO  [RS:0;d86ff69d6719:34269 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-09T21:25:04,083 WARN  [d86ff69d6719:46193 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-09T21:25:04,097 INFO  [RS:0;d86ff69d6719:34269 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C34269%2C1731187503585, suffix=, logDir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/WALs/d86ff69d6719,34269,1731187503585, archiveDir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/oldWALs, maxLogs=32
2024-11-09T21:25:04,098 INFO  [RS:0;d86ff69d6719:34269 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C34269%2C1731187503585.1731187504098
2024-11-09T21:25:04,107 INFO  [RS:0;d86ff69d6719:34269 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/WALs/d86ff69d6719,34269,1731187503585/d86ff69d6719%2C34269%2C1731187503585.1731187504098
2024-11-09T21:25:04,109 DEBUG [RS:0;d86ff69d6719:34269 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43417:43417),(127.0.0.1/127.0.0.1:42767:42767)]
2024-11-09T21:25:04,296 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:04,333 DEBUG [d86ff69d6719:46193 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-09T21:25:04,334 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d86ff69d6719,34269,1731187503585
2024-11-09T21:25:04,336 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d86ff69d6719,34269,1731187503585, state=OPENING
2024-11-09T21:25:04,365 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-09T21:25:04,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:04,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:04,383 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:25:04,383 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:25:04,383 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-09T21:25:04,384 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d86ff69d6719,34269,1731187503585}]
2024-11-09T21:25:04,537 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-09T21:25:04,538 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50155, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-09T21:25:04,541 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-09T21:25:04,541 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:25:04,543 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C34269%2C1731187503585.meta, suffix=.meta, logDir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/WALs/d86ff69d6719,34269,1731187503585, archiveDir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/oldWALs, maxLogs=32
2024-11-09T21:25:04,544 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C34269%2C1731187503585.meta.1731187504544.meta
2024-11-09T21:25:04,550 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/WALs/d86ff69d6719,34269,1731187503585/d86ff69d6719%2C34269%2C1731187503585.meta.1731187504544.meta
2024-11-09T21:25:04,551 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43417:43417),(127.0.0.1/127.0.0.1:42767:42767)]
2024-11-09T21:25:04,552 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:25:04,552 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-09T21:25:04,552 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-09T21:25:04,552 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-09T21:25:04,552 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-09T21:25:04,552 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:25:04,552 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-09T21:25:04,552 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-09T21:25:04,553 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:04,554 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-09T21:25:04,555 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-09T21:25:04,555 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:04,555 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:04,555 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-09T21:25:04,556 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-09T21:25:04,556 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:04,556 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:04,556 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-09T21:25:04,557 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-09T21:25:04,557 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:04,557 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:04,557 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-09T21:25:04,558 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-09T21:25:04,558 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:04,558 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:04,558 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-09T21:25:04,559 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740
2024-11-09T21:25:04,560 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740
2024-11-09T21:25:04,560 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:04,561 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-09T21:25:04,561 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-09T21:25:04,562 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-09T21:25:04,563 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-09T21:25:04,564 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=763819, jitterRate=-0.02875448763370514}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-09T21:25:04,564 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-09T21:25:04,564 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731187504553Writing region info on filesystem at 1731187504553Initializing all the Stores at 1731187504553Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187504553Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187504554 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187504554Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187504554Cleaning up temporary data from old regions at 1731187504561 (+7 ms)Running coprocessor post-open hooks at 1731187504564 (+3 ms)Region opened successfully at 1731187504564
2024-11-09T21:25:04,565 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731187504537
2024-11-09T21:25:04,568 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-09T21:25:04,568 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-09T21:25:04,569 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,34269,1731187503585
2024-11-09T21:25:04,569 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d86ff69d6719,34269,1731187503585, state=OPEN
2024-11-09T21:25:04,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-09T21:25:04,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-09T21:25:04,636 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d86ff69d6719,34269,1731187503585
2024-11-09T21:25:04,636 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:25:04,636 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:25:04,642 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-09T21:25:04,642 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d86ff69d6719,34269,1731187503585 in 253 msec
2024-11-09T21:25:04,648 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-09T21:25:04,648 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 712 msec
2024-11-09T21:25:04,649 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:25:04,649 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-09T21:25:04,650 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-09T21:25:04,650 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,34269,1731187503585, seqNum=-1]
2024-11-09T21:25:04,651 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T21:25:04,652 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59295, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T21:25:04,659 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 768 msec
2024-11-09T21:25:04,659 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731187504659, completionTime=-1
2024-11-09T21:25:04,659 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-09T21:25:04,659 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-09T21:25:04,661 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-09T21:25:04,661 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731187564661
2024-11-09T21:25:04,661 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731187624661
2024-11-09T21:25:04,661 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-09T21:25:04,662 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46193,1731187503426-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:04,662 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46193,1731187503426-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:04,662 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46193,1731187503426-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:04,662 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d86ff69d6719:46193, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:04,662 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:04,662 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:04,664 DEBUG [master/d86ff69d6719:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-09T21:25:04,667 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.016sec
2024-11-09T21:25:04,667 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-09T21:25:04,667 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-09T21:25:04,667 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-09T21:25:04,667 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-09T21:25:04,667 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-09T21:25:04,667 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46193,1731187503426-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T21:25:04,667 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46193,1731187503426-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-09T21:25:04,670 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-09T21:25:04,671 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-09T21:25:04,671 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46193,1731187503426-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:04,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42b3ff72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:25:04,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d86ff69d6719,46193,-1 for getting cluster id
2024-11-09T21:25:04,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-09T21:25:04,712 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e091e1c-2fe4-4522-9a15-da6501bf3fb0'
2024-11-09T21:25:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-09T21:25:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e091e1c-2fe4-4522-9a15-da6501bf3fb0"

2024-11-09T21:25:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10db42ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:25:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d86ff69d6719,46193,-1]
2024-11-09T21:25:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-09T21:25:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:04,715 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-09T21:25:04,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec96e43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:25:04,716 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-09T21:25:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,34269,1731187503585, seqNum=-1]
2024-11-09T21:25:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T21:25:04,720 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T21:25:04,722 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d86ff69d6719,46193,1731187503426
2024-11-09T21:25:04,723 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:25:04,726 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-09T21:25:04,726 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-09T21:25:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is d86ff69d6719,46193,1731187503426
2024-11-09T21:25:04,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@d9c0694
2024-11-09T21:25:04,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-09T21:25:04,729 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-09T21:25:04,730 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46193 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-09T21:25:04,730 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46193 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-09T21:25:04,730 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46193 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-09T21:25:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46193 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-11-09T21:25:04,733 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-09T21:25:04,733 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:04,733 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46193 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-11-09T21:25:04,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46193 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-09T21:25:04,734 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-09T21:25:04,739 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741835_1011 (size=381)
2024-11-09T21:25:04,740 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741835_1011 (size=381)
2024-11-09T21:25:04,741 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d3a6c7157ea5ef8d32109ac560fabda4, NAME => 'TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11
2024-11-09T21:25:04,746 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741836_1012 (size=64)
2024-11-09T21:25:04,747 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741836_1012 (size=64)
2024-11-09T21:25:04,747 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:25:04,747 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing d3a6c7157ea5ef8d32109ac560fabda4, disabling compactions & flushes
2024-11-09T21:25:04,747 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.
2024-11-09T21:25:04,747 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.
2024-11-09T21:25:04,747 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4. after waiting 0 ms
2024-11-09T21:25:04,747 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.
2024-11-09T21:25:04,747 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.
2024-11-09T21:25:04,747 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for d3a6c7157ea5ef8d32109ac560fabda4:
Waiting for close lock at 1731187504747Disabling compacts and flushes for region at 1731187504747Disabling writes for close at 1731187504747Writing region close event to WAL at 1731187504747Closed at 1731187504747
2024-11-09T21:25:04,749 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-09T21:25:04,749 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731187504749"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731187504749"}]},"ts":"1731187504749"}
2024-11-09T21:25:04,751 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-09T21:25:04,752 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-09T21:25:04,752 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187504752"}]},"ts":"1731187504752"}
2024-11-09T21:25:04,754 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-11-09T21:25:04,754 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d3a6c7157ea5ef8d32109ac560fabda4, ASSIGN}]
2024-11-09T21:25:04,755 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d3a6c7157ea5ef8d32109ac560fabda4, ASSIGN
2024-11-09T21:25:04,756 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d3a6c7157ea5ef8d32109ac560fabda4, ASSIGN; state=OFFLINE, location=d86ff69d6719,34269,1731187503585; forceNewPlan=false, retain=false
2024-11-09T21:25:04,907 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d3a6c7157ea5ef8d32109ac560fabda4, regionState=OPENING, regionLocation=d86ff69d6719,34269,1731187503585
2024-11-09T21:25:04,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d3a6c7157ea5ef8d32109ac560fabda4, ASSIGN because future has completed
2024-11-09T21:25:04,912 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3a6c7157ea5ef8d32109ac560fabda4, server=d86ff69d6719,34269,1731187503585}]
2024-11-09T21:25:05,077 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.
2024-11-09T21:25:05,078 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d3a6c7157ea5ef8d32109ac560fabda4, NAME => 'TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:25:05,079 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:05,079 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:25:05,079 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:05,079 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:05,081 INFO  [StoreOpener-d3a6c7157ea5ef8d32109ac560fabda4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d3a6c7157ea5ef8d32109ac560fabda4 
2024-11-09T21:25:05,082 INFO  [StoreOpener-d3a6c7157ea5ef8d32109ac560fabda4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3a6c7157ea5ef8d32109ac560fabda4 columnFamilyName info
2024-11-09T21:25:05,082 DEBUG [StoreOpener-d3a6c7157ea5ef8d32109ac560fabda4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:05,083 INFO  [StoreOpener-d3a6c7157ea5ef8d32109ac560fabda4-1 {}] regionserver.HStore(327): Store=d3a6c7157ea5ef8d32109ac560fabda4/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:25:05,083 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:05,084 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:05,084 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:05,085 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:05,085 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:05,086 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:05,089 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:25:05,089 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d3a6c7157ea5ef8d32109ac560fabda4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=723203, jitterRate=-0.0804009884595871}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-09T21:25:05,089 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:05,090 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d3a6c7157ea5ef8d32109ac560fabda4:
Running coprocessor pre-open hook at 1731187505079Writing region info on filesystem at 1731187505079Initializing all the Stores at 1731187505081 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187505081Cleaning up temporary data from old regions at 1731187505085 (+4 ms)Running coprocessor post-open hooks at 1731187505089 (+4 ms)Region opened successfully at 1731187505090 (+1 ms)
2024-11-09T21:25:05,091 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4., pid=6, masterSystemTime=1731187505067
2024-11-09T21:25:05,094 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.
2024-11-09T21:25:05,094 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.
2024-11-09T21:25:05,095 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d3a6c7157ea5ef8d32109ac560fabda4, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,34269,1731187503585
2024-11-09T21:25:05,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3a6c7157ea5ef8d32109ac560fabda4, server=d86ff69d6719,34269,1731187503585 because future has completed
2024-11-09T21:25:05,102 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-09T21:25:05,102 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d3a6c7157ea5ef8d32109ac560fabda4, server=d86ff69d6719,34269,1731187503585 in 187 msec
2024-11-09T21:25:05,106 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-09T21:25:05,106 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d3a6c7157ea5ef8d32109ac560fabda4, ASSIGN in 348 msec
2024-11-09T21:25:05,107 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-09T21:25:05,107 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187505107"}]},"ts":"1731187505107"}
2024-11-09T21:25:05,110 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-11-09T21:25:05,111 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-09T21:25:05,113 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 381 msec
2024-11-09T21:25:05,296 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:05,553 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:05,561 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:06,131 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,131 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,132 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,132 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,133 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,133 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,135 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,135 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,155 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,156 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,156 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,156 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,156 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,156 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,159 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,159 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,159 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,161 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,297 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:06,555 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:06,562 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:06,666 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-09T21:25:06,668 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,668 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,669 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,669 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,670 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,670 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,672 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,672 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,692 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,693 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,693 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,693 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,693 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,694 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,697 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,697 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,697 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:06,699 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:07,298 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:07,555 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:07,563 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:08,299 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:08,557 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:08,564 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:09,300 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:09,558 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:09,564 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-09T21:25:09,564 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-09T21:25:09,565 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-09T21:25:09,565 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:09,962 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-09T21:25:09,964 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-11-09T21:25:10,301 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:10,559 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:10,566 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:11,302 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:11,560 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:11,567 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:12,303 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:12,561 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:12,568 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:13,305 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:13,563 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:13,569 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:14,306 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:14,564 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:14,570 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46193 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-09T21:25:14,764 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-11-09T21:25:14,765 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-11-09T21:25:14,771 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-11-09T21:25:14,771 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.
2024-11-09T21:25:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4., hostname=d86ff69d6719,34269,1731187503585, seqNum=2]
2024-11-09T21:25:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:14,791 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3a6c7157ea5ef8d32109ac560fabda4 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-09T21:25:14,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/545db5240ade47a48e29e3efb50cdb0e is 1080, key is row0001/info:/1731187514777/Put/seqid=0
2024-11-09T21:25:14,818 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741837_1013 (size=12509)
2024-11-09T21:25:14,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741837_1013 (size=12509)
2024-11-09T21:25:14,819 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/545db5240ade47a48e29e3efb50cdb0e
2024-11-09T21:25:14,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/545db5240ade47a48e29e3efb50cdb0e as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/545db5240ade47a48e29e3efb50cdb0e
2024-11-09T21:25:14,832 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/545db5240ade47a48e29e3efb50cdb0e, entries=7, sequenceid=11, filesize=12.2 K
2024-11-09T21:25:14,833 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=19.96 KB/20444 for d3a6c7157ea5ef8d32109ac560fabda4 in 42ms, sequenceid=11, compaction requested=false
2024-11-09T21:25:14,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3a6c7157ea5ef8d32109ac560fabda4:

2024-11-09T21:25:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:14,834 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3a6c7157ea5ef8d32109ac560fabda4 1/1 column families, dataSize=22.07 KB heapSize=23.88 KB
2024-11-09T21:25:14,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/ad31ed0d4d304276a0bdd2247e6fa388 is 1080, key is row0008/info:/1731187514792/Put/seqid=0
2024-11-09T21:25:14,844 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741838_1014 (size=27607)
2024-11-09T21:25:14,844 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741838_1014 (size=27607)
2024-11-09T21:25:14,844 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.07 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/ad31ed0d4d304276a0bdd2247e6fa388
2024-11-09T21:25:14,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/ad31ed0d4d304276a0bdd2247e6fa388 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/ad31ed0d4d304276a0bdd2247e6fa388
2024-11-09T21:25:14,855 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/ad31ed0d4d304276a0bdd2247e6fa388, entries=21, sequenceid=35, filesize=27.0 K
2024-11-09T21:25:14,856 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~22.07 KB/22596, heapSize ~23.86 KB/24432, currentSize=4.20 KB/4304 for d3a6c7157ea5ef8d32109ac560fabda4 in 22ms, sequenceid=35, compaction requested=false
2024-11-09T21:25:14,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3a6c7157ea5ef8d32109ac560fabda4:

2024-11-09T21:25:14,856 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.2 K, sizeToCheck=16.0 K
2024-11-09T21:25:14,856 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:25:14,856 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/ad31ed0d4d304276a0bdd2247e6fa388 because midkey is the same as first or last row
2024-11-09T21:25:15,070 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-09T21:25:15,072 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,072 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,072 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,073 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,073 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,073 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,075 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,075 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,098 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,098 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,098 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,099 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,099 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,099 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,103 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,103 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,103 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,105 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:15,306 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:15,565 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:15,571 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:16,307 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:16,566 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:16,572 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:16,855 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3a6c7157ea5ef8d32109ac560fabda4 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-09T21:25:16,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/f3d116b6fd7c4f5ea2593ac231d12533 is 1080, key is row0029/info:/1731187514835/Put/seqid=0
2024-11-09T21:25:16,863 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741839_1015 (size=12509)
2024-11-09T21:25:16,863 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741839_1015 (size=12509)
2024-11-09T21:25:16,864 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/f3d116b6fd7c4f5ea2593ac231d12533
2024-11-09T21:25:16,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/f3d116b6fd7c4f5ea2593ac231d12533 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/f3d116b6fd7c4f5ea2593ac231d12533
2024-11-09T21:25:16,876 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/f3d116b6fd7c4f5ea2593ac231d12533, entries=7, sequenceid=45, filesize=12.2 K
2024-11-09T21:25:16,877 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for d3a6c7157ea5ef8d32109ac560fabda4 in 22ms, sequenceid=45, compaction requested=true
2024-11-09T21:25:16,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3a6c7157ea5ef8d32109ac560fabda4:

2024-11-09T21:25:16,877 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=51.4 K, sizeToCheck=16.0 K
2024-11-09T21:25:16,877 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:25:16,877 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/ad31ed0d4d304276a0bdd2247e6fa388 because midkey is the same as first or last row
2024-11-09T21:25:16,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a6c7157ea5ef8d32109ac560fabda4:info, priority=-2147483648, current under compaction store size is 1
2024-11-09T21:25:16,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:16,877 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-09T21:25:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:16,878 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3a6c7157ea5ef8d32109ac560fabda4 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-09T21:25:16,879 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 52625 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-09T21:25:16,879 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1541): d3a6c7157ea5ef8d32109ac560fabda4/info is initiating minor compaction (all files)
2024-11-09T21:25:16,879 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a6c7157ea5ef8d32109ac560fabda4/info in TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.
2024-11-09T21:25:16,879 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/545db5240ade47a48e29e3efb50cdb0e, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/ad31ed0d4d304276a0bdd2247e6fa388, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/f3d116b6fd7c4f5ea2593ac231d12533] into tmpdir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp, totalSize=51.4 K
2024-11-09T21:25:16,879 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 545db5240ade47a48e29e3efb50cdb0e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731187514777
2024-11-09T21:25:16,880 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad31ed0d4d304276a0bdd2247e6fa388, keycount=21, bloomtype=ROW, size=27.0 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1731187514792
2024-11-09T21:25:16,880 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3d116b6fd7c4f5ea2593ac231d12533, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731187514835
2024-11-09T21:25:16,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/45850639d31d437fb3a7943a2bdcd8d8 is 1080, key is row0036/info:/1731187516856/Put/seqid=0
2024-11-09T21:25:16,887 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741840_1016 (size=17894)
2024-11-09T21:25:16,887 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741840_1016 (size=17894)
2024-11-09T21:25:16,888 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/45850639d31d437fb3a7943a2bdcd8d8
2024-11-09T21:25:16,894 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a6c7157ea5ef8d32109ac560fabda4#info#compaction#59 average throughput is 17.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-09T21:25:16,895 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/265511323efb47b59d158c9cb1329146 is 1080, key is row0001/info:/1731187514777/Put/seqid=0
2024-11-09T21:25:16,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/45850639d31d437fb3a7943a2bdcd8d8 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/45850639d31d437fb3a7943a2bdcd8d8
2024-11-09T21:25:16,903 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/45850639d31d437fb3a7943a2bdcd8d8, entries=12, sequenceid=60, filesize=17.5 K
2024-11-09T21:25:16,904 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=11.56 KB/11836 for d3a6c7157ea5ef8d32109ac560fabda4 in 26ms, sequenceid=60, compaction requested=false
2024-11-09T21:25:16,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3a6c7157ea5ef8d32109ac560fabda4:

2024-11-09T21:25:16,904 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=68.9 K, sizeToCheck=16.0 K
2024-11-09T21:25:16,904 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:25:16,904 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/ad31ed0d4d304276a0bdd2247e6fa388 because midkey is the same as first or last row
2024-11-09T21:25:16,906 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741841_1017 (size=42824)
2024-11-09T21:25:16,906 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741841_1017 (size=42824)
2024-11-09T21:25:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:16,907 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3a6c7157ea5ef8d32109ac560fabda4 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-09T21:25:16,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/48a9f9d1b5964438b22eda05807ed216 is 1080, key is row0048/info:/1731187516880/Put/seqid=0
2024-11-09T21:25:16,913 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/265511323efb47b59d158c9cb1329146 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/265511323efb47b59d158c9cb1329146
2024-11-09T21:25:16,920 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741842_1018 (size=18987)
2024-11-09T21:25:16,920 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741842_1018 (size=18987)
2024-11-09T21:25:16,920 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a6c7157ea5ef8d32109ac560fabda4/info of d3a6c7157ea5ef8d32109ac560fabda4 into 265511323efb47b59d158c9cb1329146(size=41.8 K), total size for store is 59.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-09T21:25:16,920 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a6c7157ea5ef8d32109ac560fabda4:

2024-11-09T21:25:16,920 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4., storeName=d3a6c7157ea5ef8d32109ac560fabda4/info, priority=13, startTime=1731187516877; duration=0sec
2024-11-09T21:25:16,921 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K
2024-11-09T21:25:16,921 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:25:16,921 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/265511323efb47b59d158c9cb1329146 because midkey is the same as first or last row
2024-11-09T21:25:16,921 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K
2024-11-09T21:25:16,921 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:25:16,921 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/265511323efb47b59d158c9cb1329146 because midkey is the same as first or last row
2024-11-09T21:25:16,921 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K
2024-11-09T21:25:16,921 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:25:16,921 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/265511323efb47b59d158c9cb1329146 because midkey is the same as first or last row
2024-11-09T21:25:16,921 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:16,921 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a6c7157ea5ef8d32109ac560fabda4:info
2024-11-09T21:25:16,921 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/48a9f9d1b5964438b22eda05807ed216
2024-11-09T21:25:16,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/48a9f9d1b5964438b22eda05807ed216 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/48a9f9d1b5964438b22eda05807ed216
2024-11-09T21:25:16,932 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/48a9f9d1b5964438b22eda05807ed216, entries=13, sequenceid=76, filesize=18.5 K
2024-11-09T21:25:16,933 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=4.20 KB/4304 for d3a6c7157ea5ef8d32109ac560fabda4 in 26ms, sequenceid=76, compaction requested=true
2024-11-09T21:25:16,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3a6c7157ea5ef8d32109ac560fabda4:

2024-11-09T21:25:16,933 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=77.8 K, sizeToCheck=16.0 K
2024-11-09T21:25:16,934 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:25:16,934 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/265511323efb47b59d158c9cb1329146 because midkey is the same as first or last row
2024-11-09T21:25:16,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a6c7157ea5ef8d32109ac560fabda4:info, priority=-2147483648, current under compaction store size is 1
2024-11-09T21:25:16,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:16,934 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-09T21:25:16,935 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 79705 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-09T21:25:16,935 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1541): d3a6c7157ea5ef8d32109ac560fabda4/info is initiating minor compaction (all files)
2024-11-09T21:25:16,935 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a6c7157ea5ef8d32109ac560fabda4/info in TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.
2024-11-09T21:25:16,935 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/265511323efb47b59d158c9cb1329146, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/45850639d31d437fb3a7943a2bdcd8d8, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/48a9f9d1b5964438b22eda05807ed216] into tmpdir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp, totalSize=77.8 K
2024-11-09T21:25:16,936 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 265511323efb47b59d158c9cb1329146, keycount=35, bloomtype=ROW, size=41.8 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731187514777
2024-11-09T21:25:16,936 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45850639d31d437fb3a7943a2bdcd8d8, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731187516856
2024-11-09T21:25:16,936 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48a9f9d1b5964438b22eda05807ed216, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731187516880
2024-11-09T21:25:16,948 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a6c7157ea5ef8d32109ac560fabda4#info#compaction#61 average throughput is 20.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-09T21:25:16,949 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/130cd99bbd61494997276be762f9defc is 1080, key is row0001/info:/1731187514777/Put/seqid=0
2024-11-09T21:25:16,953 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741843_1019 (size=69920)
2024-11-09T21:25:16,953 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741843_1019 (size=69920)
2024-11-09T21:25:16,959 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/130cd99bbd61494997276be762f9defc as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/130cd99bbd61494997276be762f9defc
2024-11-09T21:25:16,966 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a6c7157ea5ef8d32109ac560fabda4/info of d3a6c7157ea5ef8d32109ac560fabda4 into 130cd99bbd61494997276be762f9defc(size=68.3 K), total size for store is 68.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-09T21:25:16,966 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a6c7157ea5ef8d32109ac560fabda4:

2024-11-09T21:25:16,966 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4., storeName=d3a6c7157ea5ef8d32109ac560fabda4/info, priority=13, startTime=1731187516934; duration=0sec
2024-11-09T21:25:16,966 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=68.3 K, sizeToCheck=16.0 K
2024-11-09T21:25:16,966 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:25:16,966 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/130cd99bbd61494997276be762f9defc because midkey is the same as first or last row
2024-11-09T21:25:16,967 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=68.3 K, sizeToCheck=16.0 K
2024-11-09T21:25:16,967 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:25:16,967 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/130cd99bbd61494997276be762f9defc because midkey is the same as first or last row
2024-11-09T21:25:16,967 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=68.3 K, sizeToCheck=16.0 K
2024-11-09T21:25:16,967 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:25:16,967 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/130cd99bbd61494997276be762f9defc because midkey is the same as first or last row
2024-11-09T21:25:16,967 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:16,967 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a6c7157ea5ef8d32109ac560fabda4:info
2024-11-09T21:25:17,308 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:17,567 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:17,573 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:18,309 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:18,568 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:18,574 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:18,932 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3a6c7157ea5ef8d32109ac560fabda4 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-09T21:25:18,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/5779a298e6f743049ae28ca047bb399a is 1080, key is row0061/info:/1731187516908/Put/seqid=0
2024-11-09T21:25:18,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741844_1020 (size=12509)
2024-11-09T21:25:18,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741844_1020 (size=12509)
2024-11-09T21:25:18,945 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/5779a298e6f743049ae28ca047bb399a
2024-11-09T21:25:18,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/5779a298e6f743049ae28ca047bb399a as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/5779a298e6f743049ae28ca047bb399a
2024-11-09T21:25:18,956 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/5779a298e6f743049ae28ca047bb399a, entries=7, sequenceid=88, filesize=12.2 K
2024-11-09T21:25:18,957 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for d3a6c7157ea5ef8d32109ac560fabda4 in 25ms, sequenceid=88, compaction requested=false
2024-11-09T21:25:18,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3a6c7157ea5ef8d32109ac560fabda4:

2024-11-09T21:25:18,957 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=80.5 K, sizeToCheck=16.0 K
2024-11-09T21:25:18,957 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:25:18,957 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/130cd99bbd61494997276be762f9defc because midkey is the same as first or last row
2024-11-09T21:25:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:18,959 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3a6c7157ea5ef8d32109ac560fabda4 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-09T21:25:18,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/280fd89bf0b348a28c1bdab43e092d8b is 1080, key is row0068/info:/1731187518934/Put/seqid=0
2024-11-09T21:25:18,967 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741845_1021 (size=17894)
2024-11-09T21:25:18,967 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741845_1021 (size=17894)
2024-11-09T21:25:18,968 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/280fd89bf0b348a28c1bdab43e092d8b
2024-11-09T21:25:18,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/280fd89bf0b348a28c1bdab43e092d8b as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/280fd89bf0b348a28c1bdab43e092d8b
2024-11-09T21:25:18,980 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/280fd89bf0b348a28c1bdab43e092d8b, entries=12, sequenceid=103, filesize=17.5 K
2024-11-09T21:25:18,981 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=9.46 KB/9684 for d3a6c7157ea5ef8d32109ac560fabda4 in 22ms, sequenceid=103, compaction requested=true
2024-11-09T21:25:18,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3a6c7157ea5ef8d32109ac560fabda4:

2024-11-09T21:25:18,981 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=98.0 K, sizeToCheck=16.0 K
2024-11-09T21:25:18,981 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:25:18,981 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/130cd99bbd61494997276be762f9defc because midkey is the same as first or last row
2024-11-09T21:25:18,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a6c7157ea5ef8d32109ac560fabda4:info, priority=-2147483648, current under compaction store size is 1
2024-11-09T21:25:18,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:18,981 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-09T21:25:18,982 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 100323 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-09T21:25:18,982 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1541): d3a6c7157ea5ef8d32109ac560fabda4/info is initiating minor compaction (all files)
2024-11-09T21:25:18,983 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a6c7157ea5ef8d32109ac560fabda4/info in TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.
2024-11-09T21:25:18,983 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/130cd99bbd61494997276be762f9defc, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/5779a298e6f743049ae28ca047bb399a, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/280fd89bf0b348a28c1bdab43e092d8b] into tmpdir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp, totalSize=98.0 K
2024-11-09T21:25:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:18,983 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3a6c7157ea5ef8d32109ac560fabda4 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-09T21:25:18,983 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 130cd99bbd61494997276be762f9defc, keycount=60, bloomtype=ROW, size=68.3 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731187514777
2024-11-09T21:25:18,984 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5779a298e6f743049ae28ca047bb399a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731187516908
2024-11-09T21:25:18,984 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 280fd89bf0b348a28c1bdab43e092d8b, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731187518934
2024-11-09T21:25:18,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/20b6b0208a234b759908b05ee0a3c18e is 1080, key is row0080/info:/1731187518960/Put/seqid=0
2024-11-09T21:25:18,993 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741846_1022 (size=16817)
2024-11-09T21:25:18,994 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741846_1022 (size=16817)
2024-11-09T21:25:18,994 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/20b6b0208a234b759908b05ee0a3c18e
2024-11-09T21:25:18,998 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a6c7157ea5ef8d32109ac560fabda4#info#compaction#65 average throughput is 20.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-09T21:25:18,999 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/c3b2d5b3d4644ccbaac48861a5dfee70 is 1080, key is row0001/info:/1731187514777/Put/seqid=0
2024-11-09T21:25:19,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/20b6b0208a234b759908b05ee0a3c18e as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/20b6b0208a234b759908b05ee0a3c18e
2024-11-09T21:25:19,004 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741847_1023 (size=90562)
2024-11-09T21:25:19,005 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741847_1023 (size=90562)
2024-11-09T21:25:19,006 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/20b6b0208a234b759908b05ee0a3c18e, entries=11, sequenceid=117, filesize=16.4 K
2024-11-09T21:25:19,007 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=6.30 KB/6456 for d3a6c7157ea5ef8d32109ac560fabda4 in 24ms, sequenceid=117, compaction requested=false
2024-11-09T21:25:19,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3a6c7157ea5ef8d32109ac560fabda4:

2024-11-09T21:25:19,007 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=114.4 K, sizeToCheck=16.0 K
2024-11-09T21:25:19,007 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:25:19,007 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/130cd99bbd61494997276be762f9defc because midkey is the same as first or last row
2024-11-09T21:25:19,012 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/c3b2d5b3d4644ccbaac48861a5dfee70 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/c3b2d5b3d4644ccbaac48861a5dfee70
2024-11-09T21:25:19,017 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a6c7157ea5ef8d32109ac560fabda4/info of d3a6c7157ea5ef8d32109ac560fabda4 into c3b2d5b3d4644ccbaac48861a5dfee70(size=88.4 K), total size for store is 104.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-09T21:25:19,017 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a6c7157ea5ef8d32109ac560fabda4:

2024-11-09T21:25:19,018 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4., storeName=d3a6c7157ea5ef8d32109ac560fabda4/info, priority=13, startTime=1731187518981; duration=0sec
2024-11-09T21:25:19,018 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=104.9 K, sizeToCheck=16.0 K
2024-11-09T21:25:19,018 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:25:19,018 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=104.9 K, sizeToCheck=16.0 K
2024-11-09T21:25:19,018 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:25:19,018 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=104.9 K, sizeToCheck=16.0 K
2024-11-09T21:25:19,018 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-09T21:25:19,019 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:19,019 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:19,019 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a6c7157ea5ef8d32109ac560fabda4:info
2024-11-09T21:25:19,020 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46193 {}] assignment.AssignmentManager(1363): Split request from d86ff69d6719,34269,1731187503585, parent={ENCODED => d3a6c7157ea5ef8d32109ac560fabda4, NAME => 'TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-09T21:25:19,024 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46193 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=d86ff69d6719,34269,1731187503585
2024-11-09T21:25:19,027 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46193 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=d3a6c7157ea5ef8d32109ac560fabda4, daughterA=f16719376f7f3978f93657892e985b13, daughterB=cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:19,028 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=d3a6c7157ea5ef8d32109ac560fabda4, daughterA=f16719376f7f3978f93657892e985b13, daughterB=cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:19,028 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=d3a6c7157ea5ef8d32109ac560fabda4, daughterA=f16719376f7f3978f93657892e985b13, daughterB=cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:19,028 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=d3a6c7157ea5ef8d32109ac560fabda4, daughterA=f16719376f7f3978f93657892e985b13, daughterB=cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:19,035 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d3a6c7157ea5ef8d32109ac560fabda4, UNASSIGN}]
2024-11-09T21:25:19,036 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d3a6c7157ea5ef8d32109ac560fabda4, UNASSIGN
2024-11-09T21:25:19,038 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=d3a6c7157ea5ef8d32109ac560fabda4, regionState=CLOSING, regionLocation=d86ff69d6719,34269,1731187503585
2024-11-09T21:25:19,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d3a6c7157ea5ef8d32109ac560fabda4, UNASSIGN because future has completed
2024-11-09T21:25:19,040 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-11-09T21:25:19,040 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3a6c7157ea5ef8d32109ac560fabda4, server=d86ff69d6719,34269,1731187503585}]
2024-11-09T21:25:19,196 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:19,196 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-11-09T21:25:19,196 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing d3a6c7157ea5ef8d32109ac560fabda4, disabling compactions & flushes
2024-11-09T21:25:19,196 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.
2024-11-09T21:25:19,196 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.
2024-11-09T21:25:19,197 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4. after waiting 0 ms
2024-11-09T21:25:19,197 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.
2024-11-09T21:25:19,197 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing d3a6c7157ea5ef8d32109ac560fabda4 1/1 column families, dataSize=6.30 KB heapSize=7 KB
2024-11-09T21:25:19,202 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/c3540fd1cbd44748a1ac950ed56daac0 is 1080, key is row0091/info:/1731187518985/Put/seqid=0
2024-11-09T21:25:19,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741848_1024 (size=11424)
2024-11-09T21:25:19,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741848_1024 (size=11424)
2024-11-09T21:25:19,209 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.30 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/c3540fd1cbd44748a1ac950ed56daac0
2024-11-09T21:25:19,216 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/.tmp/info/c3540fd1cbd44748a1ac950ed56daac0 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/c3540fd1cbd44748a1ac950ed56daac0
2024-11-09T21:25:19,222 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/c3540fd1cbd44748a1ac950ed56daac0, entries=6, sequenceid=127, filesize=11.2 K
2024-11-09T21:25:19,223 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~6.30 KB/6456, heapSize ~6.98 KB/7152, currentSize=0 B/0 for d3a6c7157ea5ef8d32109ac560fabda4 in 25ms, sequenceid=127, compaction requested=true
2024-11-09T21:25:19,224 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/545db5240ade47a48e29e3efb50cdb0e, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/ad31ed0d4d304276a0bdd2247e6fa388, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/265511323efb47b59d158c9cb1329146, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/f3d116b6fd7c4f5ea2593ac231d12533, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/45850639d31d437fb3a7943a2bdcd8d8, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/130cd99bbd61494997276be762f9defc, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/48a9f9d1b5964438b22eda05807ed216, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/5779a298e6f743049ae28ca047bb399a, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/280fd89bf0b348a28c1bdab43e092d8b] to archive
2024-11-09T21:25:19,225 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-09T21:25:19,226 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/545db5240ade47a48e29e3efb50cdb0e to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/545db5240ade47a48e29e3efb50cdb0e
2024-11-09T21:25:19,228 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/ad31ed0d4d304276a0bdd2247e6fa388 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/ad31ed0d4d304276a0bdd2247e6fa388
2024-11-09T21:25:19,229 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/265511323efb47b59d158c9cb1329146 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/265511323efb47b59d158c9cb1329146
2024-11-09T21:25:19,230 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/f3d116b6fd7c4f5ea2593ac231d12533 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/f3d116b6fd7c4f5ea2593ac231d12533
2024-11-09T21:25:19,231 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/45850639d31d437fb3a7943a2bdcd8d8 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/45850639d31d437fb3a7943a2bdcd8d8
2024-11-09T21:25:19,231 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/130cd99bbd61494997276be762f9defc to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/130cd99bbd61494997276be762f9defc
2024-11-09T21:25:19,232 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/48a9f9d1b5964438b22eda05807ed216 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/48a9f9d1b5964438b22eda05807ed216
2024-11-09T21:25:19,233 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/5779a298e6f743049ae28ca047bb399a to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/5779a298e6f743049ae28ca047bb399a
2024-11-09T21:25:19,234 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/280fd89bf0b348a28c1bdab43e092d8b to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/280fd89bf0b348a28c1bdab43e092d8b
2024-11-09T21:25:19,239 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=1
2024-11-09T21:25:19,240 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.
2024-11-09T21:25:19,240 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for d3a6c7157ea5ef8d32109ac560fabda4:
Waiting for close lock at 1731187519196Running coprocessor pre-close hooks at 1731187519196Disabling compacts and flushes for region at 1731187519196Disabling writes for close at 1731187519197 (+1 ms)Obtaining lock to block concurrent updates at 1731187519197Preparing flush snapshotting stores in d3a6c7157ea5ef8d32109ac560fabda4 at 1731187519197Finished memstore snapshotting TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4., syncing WAL and waiting on mvcc, flushsize=dataSize=6456, getHeapSize=7152, getOffHeapSize=0, getCellsCount=6 at 1731187519197Flushing stores of TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4. at 1731187519198 (+1 ms)Flushing d3a6c7157ea5ef8d32109ac560fabda4/info: creating writer at 1731187519198Flushing d3a6c7157ea5ef8d32109ac560fabda4/info: appending metadata at 1731187519201 (+3 ms)Flushing d3a6c7157ea5ef8d32109ac560fabda4/info: closing flushed file at 1731187519201Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79c9dcf5: reopening flushed file at 1731187519215 (+14 ms)Finished flush of dataSize ~6.30 KB/6456, heapSize ~6.98 KB/7152, currentSize=0 B/0 for d3a6c7157ea5ef8d32109ac560fabda4 in 25ms, sequenceid=127, compaction requested=true at 1731187519223 (+8 ms)Writing region close event to WAL at 1731187519236 (+13 ms)Running coprocessor post-close hooks at 1731187519240 (+4 ms)Closed at 1731187519240
2024-11-09T21:25:19,242 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:19,243 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=d3a6c7157ea5ef8d32109ac560fabda4, regionState=CLOSED
2024-11-09T21:25:19,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3a6c7157ea5ef8d32109ac560fabda4, server=d86ff69d6719,34269,1731187503585 because future has completed
2024-11-09T21:25:19,248 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-09T21:25:19,248 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure d3a6c7157ea5ef8d32109ac560fabda4, server=d86ff69d6719,34269,1731187503585 in 206 msec
2024-11-09T21:25:19,250 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-09T21:25:19,250 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d3a6c7157ea5ef8d32109ac560fabda4, UNASSIGN in 213 msec
2024-11-09T21:25:19,257 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:19,260 INFO  [PEWorker-5 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=d3a6c7157ea5ef8d32109ac560fabda4, threads=3
2024-11-09T21:25:19,262 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/20b6b0208a234b759908b05ee0a3c18e for region: d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:19,262 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/c3b2d5b3d4644ccbaac48861a5dfee70 for region: d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:19,262 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/c3540fd1cbd44748a1ac950ed56daac0 for region: d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:19,271 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/20b6b0208a234b759908b05ee0a3c18e, top=true
2024-11-09T21:25:19,271 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/c3540fd1cbd44748a1ac950ed56daac0, top=true
2024-11-09T21:25:19,280 INFO  [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/TestLogRolling-testLogRolling=d3a6c7157ea5ef8d32109ac560fabda4-20b6b0208a234b759908b05ee0a3c18e for child: cc0e2a2014f92d28c5b9a7c68250eadb, parent: d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:19,280 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741849_1025 (size=27)
2024-11-09T21:25:19,281 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/20b6b0208a234b759908b05ee0a3c18e for region: d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:19,281 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741849_1025 (size=27)
2024-11-09T21:25:19,281 INFO  [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/TestLogRolling-testLogRolling=d3a6c7157ea5ef8d32109ac560fabda4-c3540fd1cbd44748a1ac950ed56daac0 for child: cc0e2a2014f92d28c5b9a7c68250eadb, parent: d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:19,281 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/c3540fd1cbd44748a1ac950ed56daac0 for region: d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:19,290 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741850_1026 (size=27)
2024-11-09T21:25:19,290 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741850_1026 (size=27)
2024-11-09T21:25:19,290 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/c3b2d5b3d4644ccbaac48861a5dfee70 for region: d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:19,292 DEBUG [PEWorker-5 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region d3a6c7157ea5ef8d32109ac560fabda4 Daughter A: [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/f16719376f7f3978f93657892e985b13/info/c3b2d5b3d4644ccbaac48861a5dfee70.d3a6c7157ea5ef8d32109ac560fabda4] storefiles, Daughter B: [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/TestLogRolling-testLogRolling=d3a6c7157ea5ef8d32109ac560fabda4-20b6b0208a234b759908b05ee0a3c18e, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/TestLogRolling-testLogRolling=d3a6c7157ea5ef8d32109ac560fabda4-c3540fd1cbd44748a1ac950ed56daac0, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/c3b2d5b3d4644ccbaac48861a5dfee70.d3a6c7157ea5ef8d32109ac560fabda4] storefiles.
2024-11-09T21:25:19,300 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741851_1027 (size=71)
2024-11-09T21:25:19,300 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741851_1027 (size=71)
2024-11-09T21:25:19,302 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:19,310 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:19,313 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741852_1028 (size=71)
2024-11-09T21:25:19,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741852_1028 (size=71)
2024-11-09T21:25:19,315 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:19,325 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/f16719376f7f3978f93657892e985b13/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-09T21:25:19,327 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-09T21:25:19,329 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731187519329"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1731187519329"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1731187519329"}]},"ts":"1731187519329"}
2024-11-09T21:25:19,330 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731187519329"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731187519329"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731187519329"}]},"ts":"1731187519329"}
2024-11-09T21:25:19,330 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731187519329"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731187519329"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731187519329"}]},"ts":"1731187519329"}
2024-11-09T21:25:19,345 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f16719376f7f3978f93657892e985b13, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=cc0e2a2014f92d28c5b9a7c68250eadb, ASSIGN}]
2024-11-09T21:25:19,346 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f16719376f7f3978f93657892e985b13, ASSIGN
2024-11-09T21:25:19,346 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=cc0e2a2014f92d28c5b9a7c68250eadb, ASSIGN
2024-11-09T21:25:19,347 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f16719376f7f3978f93657892e985b13, ASSIGN; state=SPLITTING_NEW, location=d86ff69d6719,34269,1731187503585; forceNewPlan=false, retain=false
2024-11-09T21:25:19,347 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=cc0e2a2014f92d28c5b9a7c68250eadb, ASSIGN; state=SPLITTING_NEW, location=d86ff69d6719,34269,1731187503585; forceNewPlan=false, retain=false
2024-11-09T21:25:19,498 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=cc0e2a2014f92d28c5b9a7c68250eadb, regionState=OPENING, regionLocation=d86ff69d6719,34269,1731187503585
2024-11-09T21:25:19,498 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f16719376f7f3978f93657892e985b13, regionState=OPENING, regionLocation=d86ff69d6719,34269,1731187503585
2024-11-09T21:25:19,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f16719376f7f3978f93657892e985b13, ASSIGN because future has completed
2024-11-09T21:25:19,500 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f16719376f7f3978f93657892e985b13, server=d86ff69d6719,34269,1731187503585}]
2024-11-09T21:25:19,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=cc0e2a2014f92d28c5b9a7c68250eadb, ASSIGN because future has completed
2024-11-09T21:25:19,502 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc0e2a2014f92d28c5b9a7c68250eadb, server=d86ff69d6719,34269,1731187503585}]
2024-11-09T21:25:19,563 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-09T21:25:19,569 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:19,575 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:19,656 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.
2024-11-09T21:25:19,656 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => f16719376f7f3978f93657892e985b13, NAME => 'TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.', STARTKEY => '', ENDKEY => 'row0062'}
2024-11-09T21:25:19,657 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling f16719376f7f3978f93657892e985b13
2024-11-09T21:25:19,657 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:25:19,657 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for f16719376f7f3978f93657892e985b13
2024-11-09T21:25:19,657 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for f16719376f7f3978f93657892e985b13
2024-11-09T21:25:19,658 INFO  [StoreOpener-f16719376f7f3978f93657892e985b13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region f16719376f7f3978f93657892e985b13 
2024-11-09T21:25:19,659 INFO  [StoreOpener-f16719376f7f3978f93657892e985b13-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f16719376f7f3978f93657892e985b13 columnFamilyName info
2024-11-09T21:25:19,659 DEBUG [StoreOpener-f16719376f7f3978f93657892e985b13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:19,670 DEBUG [StoreOpener-f16719376f7f3978f93657892e985b13-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/f16719376f7f3978f93657892e985b13/info/c3b2d5b3d4644ccbaac48861a5dfee70.d3a6c7157ea5ef8d32109ac560fabda4->hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/c3b2d5b3d4644ccbaac48861a5dfee70-bottom
2024-11-09T21:25:19,671 INFO  [StoreOpener-f16719376f7f3978f93657892e985b13-1 {}] regionserver.HStore(327): Store=f16719376f7f3978f93657892e985b13/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:25:19,671 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for f16719376f7f3978f93657892e985b13
2024-11-09T21:25:19,672 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/f16719376f7f3978f93657892e985b13
2024-11-09T21:25:19,673 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/f16719376f7f3978f93657892e985b13
2024-11-09T21:25:19,674 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for f16719376f7f3978f93657892e985b13
2024-11-09T21:25:19,674 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for f16719376f7f3978f93657892e985b13
2024-11-09T21:25:19,675 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for f16719376f7f3978f93657892e985b13
2024-11-09T21:25:19,676 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened f16719376f7f3978f93657892e985b13; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=794364, jitterRate=0.01008722186088562}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-09T21:25:19,676 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f16719376f7f3978f93657892e985b13
2024-11-09T21:25:19,677 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for f16719376f7f3978f93657892e985b13:
Running coprocessor pre-open hook at 1731187519657Writing region info on filesystem at 1731187519657Initializing all the Stores at 1731187519658 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187519658Cleaning up temporary data from old regions at 1731187519674 (+16 ms)Running coprocessor post-open hooks at 1731187519676 (+2 ms)Region opened successfully at 1731187519677 (+1 ms)
2024-11-09T21:25:19,678 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13., pid=12, masterSystemTime=1731187519653
2024-11-09T21:25:19,678 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store f16719376f7f3978f93657892e985b13:info, priority=-2147483648, current under compaction store size is 1
2024-11-09T21:25:19,678 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-09T21:25:19,678 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:19,679 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.
2024-11-09T21:25:19,679 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1541): f16719376f7f3978f93657892e985b13/info is initiating minor compaction (all files)
2024-11-09T21:25:19,679 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f16719376f7f3978f93657892e985b13/info in TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.
2024-11-09T21:25:19,679 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/f16719376f7f3978f93657892e985b13/info/c3b2d5b3d4644ccbaac48861a5dfee70.d3a6c7157ea5ef8d32109ac560fabda4->hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/c3b2d5b3d4644ccbaac48861a5dfee70-bottom] into tmpdir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/f16719376f7f3978f93657892e985b13/.tmp, totalSize=88.4 K
2024-11-09T21:25:19,680 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3b2d5b3d4644ccbaac48861a5dfee70.d3a6c7157ea5ef8d32109ac560fabda4, keycount=39, bloomtype=ROW, size=88.4 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731187514777
2024-11-09T21:25:19,680 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.
2024-11-09T21:25:19,680 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.
2024-11-09T21:25:19,680 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:19,680 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => cc0e2a2014f92d28c5b9a7c68250eadb, NAME => 'TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.', STARTKEY => 'row0062', ENDKEY => ''}
2024-11-09T21:25:19,681 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:19,681 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:25:19,681 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:19,681 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:19,681 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f16719376f7f3978f93657892e985b13, regionState=OPEN, openSeqNum=131, regionLocation=d86ff69d6719,34269,1731187503585
2024-11-09T21:25:19,682 INFO  [StoreOpener-cc0e2a2014f92d28c5b9a7c68250eadb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region cc0e2a2014f92d28c5b9a7c68250eadb 
2024-11-09T21:25:19,683 INFO  [StoreOpener-cc0e2a2014f92d28c5b9a7c68250eadb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc0e2a2014f92d28c5b9a7c68250eadb columnFamilyName info
2024-11-09T21:25:19,683 DEBUG [StoreOpener-cc0e2a2014f92d28c5b9a7c68250eadb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:19,683 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-11-09T21:25:19,683 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-11-09T21:25:19,683 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB
2024-11-09T21:25:19,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f16719376f7f3978f93657892e985b13, server=d86ff69d6719,34269,1731187503585 because future has completed
2024-11-09T21:25:19,688 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10
2024-11-09T21:25:19,688 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure f16719376f7f3978f93657892e985b13, server=d86ff69d6719,34269,1731187503585 in 185 msec
2024-11-09T21:25:19,690 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f16719376f7f3978f93657892e985b13, ASSIGN in 343 msec
2024-11-09T21:25:19,692 DEBUG [StoreOpener-cc0e2a2014f92d28c5b9a7c68250eadb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/TestLogRolling-testLogRolling=d3a6c7157ea5ef8d32109ac560fabda4-20b6b0208a234b759908b05ee0a3c18e
2024-11-09T21:25:19,697 DEBUG [StoreOpener-cc0e2a2014f92d28c5b9a7c68250eadb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/TestLogRolling-testLogRolling=d3a6c7157ea5ef8d32109ac560fabda4-c3540fd1cbd44748a1ac950ed56daac0
2024-11-09T21:25:19,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/.tmp/info/e4f36d1627694b45bee403e005c4f275 is 193, key is TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb./info:regioninfo/1731187519497/Put/seqid=0
2024-11-09T21:25:19,701 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f16719376f7f3978f93657892e985b13#info#compaction#67 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-09T21:25:19,701 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/f16719376f7f3978f93657892e985b13/.tmp/info/fa70e4ce31aa4371b27fc5fab1905977 is 1080, key is row0001/info:/1731187514777/Put/seqid=0
2024-11-09T21:25:19,702 DEBUG [StoreOpener-cc0e2a2014f92d28c5b9a7c68250eadb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/c3b2d5b3d4644ccbaac48861a5dfee70.d3a6c7157ea5ef8d32109ac560fabda4->hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/c3b2d5b3d4644ccbaac48861a5dfee70-top
2024-11-09T21:25:19,702 INFO  [StoreOpener-cc0e2a2014f92d28c5b9a7c68250eadb-1 {}] regionserver.HStore(327): Store=cc0e2a2014f92d28c5b9a7c68250eadb/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:25:19,702 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:19,703 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741853_1029 (size=9847)
2024-11-09T21:25:19,703 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741853_1029 (size=9847)
2024-11-09T21:25:19,703 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:19,704 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/.tmp/info/e4f36d1627694b45bee403e005c4f275
2024-11-09T21:25:19,704 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:19,705 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:19,705 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:19,706 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741854_1030 (size=70862)
2024-11-09T21:25:19,706 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741854_1030 (size=70862)
2024-11-09T21:25:19,707 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:19,708 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened cc0e2a2014f92d28c5b9a7c68250eadb; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=737633, jitterRate=-0.06205189228057861}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-09T21:25:19,708 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:19,708 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for cc0e2a2014f92d28c5b9a7c68250eadb:
Running coprocessor pre-open hook at 1731187519681Writing region info on filesystem at 1731187519681Initializing all the Stores at 1731187519681Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187519681Cleaning up temporary data from old regions at 1731187519705 (+24 ms)Running coprocessor post-open hooks at 1731187519708 (+3 ms)Region opened successfully at 1731187519708
2024-11-09T21:25:19,709 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb., pid=13, masterSystemTime=1731187519653
2024-11-09T21:25:19,709 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store cc0e2a2014f92d28c5b9a7c68250eadb:info, priority=-2147483648, current under compaction store size is 2
2024-11-09T21:25:19,709 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:19,709 DEBUG [RS:0;d86ff69d6719:34269-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-09T21:25:19,710 INFO  [RS:0;d86ff69d6719:34269-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:19,710 DEBUG [RS:0;d86ff69d6719:34269-longCompactions-0 {}] regionserver.HStore(1541): cc0e2a2014f92d28c5b9a7c68250eadb/info is initiating minor compaction (all files)
2024-11-09T21:25:19,710 INFO  [RS:0;d86ff69d6719:34269-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc0e2a2014f92d28c5b9a7c68250eadb/info in TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:19,710 INFO  [RS:0;d86ff69d6719:34269-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/c3b2d5b3d4644ccbaac48861a5dfee70.d3a6c7157ea5ef8d32109ac560fabda4->hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/c3b2d5b3d4644ccbaac48861a5dfee70-top, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/TestLogRolling-testLogRolling=d3a6c7157ea5ef8d32109ac560fabda4-20b6b0208a234b759908b05ee0a3c18e, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/TestLogRolling-testLogRolling=d3a6c7157ea5ef8d32109ac560fabda4-c3540fd1cbd44748a1ac950ed56daac0] into tmpdir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp, totalSize=116.0 K
2024-11-09T21:25:19,711 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:19,711 INFO  [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:19,711 DEBUG [RS:0;d86ff69d6719:34269-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b2d5b3d4644ccbaac48861a5dfee70.d3a6c7157ea5ef8d32109ac560fabda4, keycount=39, bloomtype=ROW, size=88.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731187514777
2024-11-09T21:25:19,711 DEBUG [RS:0;d86ff69d6719:34269-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=d3a6c7157ea5ef8d32109ac560fabda4-20b6b0208a234b759908b05ee0a3c18e, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731187518960
2024-11-09T21:25:19,712 DEBUG [RS:0;d86ff69d6719:34269-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=d3a6c7157ea5ef8d32109ac560fabda4-c3540fd1cbd44748a1ac950ed56daac0, keycount=6, bloomtype=ROW, size=11.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731187518985
2024-11-09T21:25:19,712 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=cc0e2a2014f92d28c5b9a7c68250eadb, regionState=OPEN, openSeqNum=131, regionLocation=d86ff69d6719,34269,1731187503585
2024-11-09T21:25:19,712 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/f16719376f7f3978f93657892e985b13/.tmp/info/fa70e4ce31aa4371b27fc5fab1905977 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/f16719376f7f3978f93657892e985b13/info/fa70e4ce31aa4371b27fc5fab1905977
2024-11-09T21:25:19,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc0e2a2014f92d28c5b9a7c68250eadb, server=d86ff69d6719,34269,1731187503585 because future has completed
2024-11-09T21:25:19,719 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=11
2024-11-09T21:25:19,719 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure cc0e2a2014f92d28c5b9a7c68250eadb, server=d86ff69d6719,34269,1731187503585 in 214 msec
2024-11-09T21:25:19,719 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in f16719376f7f3978f93657892e985b13/info of f16719376f7f3978f93657892e985b13 into fa70e4ce31aa4371b27fc5fab1905977(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-09T21:25:19,719 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f16719376f7f3978f93657892e985b13:

2024-11-09T21:25:19,719 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13., storeName=f16719376f7f3978f93657892e985b13/info, priority=15, startTime=1731187519678; duration=0sec
2024-11-09T21:25:19,719 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:19,719 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f16719376f7f3978f93657892e985b13:info
2024-11-09T21:25:19,721 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7
2024-11-09T21:25:19,721 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=cc0e2a2014f92d28c5b9a7c68250eadb, ASSIGN in 374 msec
2024-11-09T21:25:19,723 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=d3a6c7157ea5ef8d32109ac560fabda4, daughterA=f16719376f7f3978f93657892e985b13, daughterB=cc0e2a2014f92d28c5b9a7c68250eadb in 697 msec
2024-11-09T21:25:19,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/.tmp/ns/3067c0c5c7874805b5e0ff42d16e62e4 is 43, key is default/ns:d/1731187504653/Put/seqid=0
2024-11-09T21:25:19,729 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741855_1031 (size=5153)
2024-11-09T21:25:19,729 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741855_1031 (size=5153)
2024-11-09T21:25:19,730 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/.tmp/ns/3067c0c5c7874805b5e0ff42d16e62e4
2024-11-09T21:25:19,733 INFO  [RS:0;d86ff69d6719:34269-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc0e2a2014f92d28c5b9a7c68250eadb#info#compaction#70 average throughput is 35.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-09T21:25:19,734 DEBUG [RS:0;d86ff69d6719:34269-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/1e6eacdd04044f31a92f6ee33108d146 is 1080, key is row0062/info:/1731187516911/Put/seqid=0
2024-11-09T21:25:19,738 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741856_1032 (size=42984)
2024-11-09T21:25:19,738 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741856_1032 (size=42984)
2024-11-09T21:25:19,744 DEBUG [RS:0;d86ff69d6719:34269-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/1e6eacdd04044f31a92f6ee33108d146 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/1e6eacdd04044f31a92f6ee33108d146
2024-11-09T21:25:19,750 INFO  [RS:0;d86ff69d6719:34269-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc0e2a2014f92d28c5b9a7c68250eadb/info of cc0e2a2014f92d28c5b9a7c68250eadb into 1e6eacdd04044f31a92f6ee33108d146(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-09T21:25:19,750 DEBUG [RS:0;d86ff69d6719:34269-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:19,750 INFO  [RS:0;d86ff69d6719:34269-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb., storeName=cc0e2a2014f92d28c5b9a7c68250eadb/info, priority=13, startTime=1731187519709; duration=0sec
2024-11-09T21:25:19,750 DEBUG [RS:0;d86ff69d6719:34269-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:19,750 DEBUG [RS:0;d86ff69d6719:34269-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc0e2a2014f92d28c5b9a7c68250eadb:info
2024-11-09T21:25:19,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/.tmp/table/249b45de47014e15bc055836c5c7dd3e is 65, key is TestLogRolling-testLogRolling/table:state/1731187505107/Put/seqid=0
2024-11-09T21:25:19,760 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741857_1033 (size=5340)
2024-11-09T21:25:19,760 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741857_1033 (size=5340)
2024-11-09T21:25:19,761 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/.tmp/table/249b45de47014e15bc055836c5c7dd3e
2024-11-09T21:25:19,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/.tmp/info/e4f36d1627694b45bee403e005c4f275 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/info/e4f36d1627694b45bee403e005c4f275
2024-11-09T21:25:19,772 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/info/e4f36d1627694b45bee403e005c4f275, entries=30, sequenceid=17, filesize=9.6 K
2024-11-09T21:25:19,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/.tmp/ns/3067c0c5c7874805b5e0ff42d16e62e4 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/ns/3067c0c5c7874805b5e0ff42d16e62e4
2024-11-09T21:25:19,778 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/ns/3067c0c5c7874805b5e0ff42d16e62e4, entries=2, sequenceid=17, filesize=5.0 K
2024-11-09T21:25:19,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/.tmp/table/249b45de47014e15bc055836c5c7dd3e as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/table/249b45de47014e15bc055836c5c7dd3e
2024-11-09T21:25:19,785 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/table/249b45de47014e15bc055836c5c7dd3e, entries=2, sequenceid=17, filesize=5.2 K
2024-11-09T21:25:19,786 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 102ms, sequenceid=17, compaction requested=false
2024-11-09T21:25:19,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-09T21:25:20,311 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:20,570 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:20,576 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:36704 deadline: 1731187530999, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4. is not online on d86ff69d6719,34269,1731187503585
2024-11-09T21:25:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4., hostname=d86ff69d6719,34269,1731187503585, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4., hostname=d86ff69d6719,34269,1731187503585, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4. is not online on d86ff69d6719,34269,1731187503585
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-09T21:25:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4., hostname=d86ff69d6719,34269,1731187503585, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4. is not online on d86ff69d6719,34269,1731187503585
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-09T21:25:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1731187504730.d3a6c7157ea5ef8d32109ac560fabda4., hostname=d86ff69d6719,34269,1731187503585, seqNum=2 from cache
2024-11-09T21:25:21,312 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:21,571 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:21,577 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:22,313 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:22,573 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:22,578 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:23,314 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:23,574 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:23,578 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:24,315 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:24,575 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:24,579 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:24,744 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-09T21:25:24,746 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,746 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,747 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,748 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,748 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,749 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,750 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,751 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,770 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,775 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,775 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,775 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:24,777 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-09T21:25:25,315 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:25,576 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:25,580 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:26,316 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:26,578 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:26,581 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:27,317 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:27,579 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:27,582 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:28,318 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:28,580 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:28,583 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:29,319 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:29,563 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-09T21:25:29,563 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-09T21:25:29,581 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:29,583 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:30,320 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:30,581 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:30,584 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:31,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb., hostname=d86ff69d6719,34269,1731187503585, seqNum=131]
2024-11-09T21:25:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:31,149 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc0e2a2014f92d28c5b9a7c68250eadb 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-09T21:25:31,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/4971793ab1cd407f934763ad884dbf2b is 1080, key is row0097/info:/1731187531139/Put/seqid=0
2024-11-09T21:25:31,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741858_1034 (size=12516)
2024-11-09T21:25:31,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741858_1034 (size=12516)
2024-11-09T21:25:31,159 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/4971793ab1cd407f934763ad884dbf2b
2024-11-09T21:25:31,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/4971793ab1cd407f934763ad884dbf2b as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/4971793ab1cd407f934763ad884dbf2b
2024-11-09T21:25:31,171 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/4971793ab1cd407f934763ad884dbf2b, entries=7, sequenceid=141, filesize=12.2 K
2024-11-09T21:25:31,173 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for cc0e2a2014f92d28c5b9a7c68250eadb in 23ms, sequenceid=141, compaction requested=false
2024-11-09T21:25:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:31,174 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc0e2a2014f92d28c5b9a7c68250eadb 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-09T21:25:31,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/dec70af8d06c4f3db17b46967cff5f40 is 1080, key is row0104/info:/1731187531150/Put/seqid=0
2024-11-09T21:25:31,184 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741859_1035 (size=19000)
2024-11-09T21:25:31,184 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741859_1035 (size=19000)
2024-11-09T21:25:31,184 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/dec70af8d06c4f3db17b46967cff5f40
2024-11-09T21:25:31,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/dec70af8d06c4f3db17b46967cff5f40 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/dec70af8d06c4f3db17b46967cff5f40
2024-11-09T21:25:31,196 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/dec70af8d06c4f3db17b46967cff5f40, entries=13, sequenceid=157, filesize=18.6 K
2024-11-09T21:25:31,197 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=11.56 KB/11836 for cc0e2a2014f92d28c5b9a7c68250eadb in 23ms, sequenceid=157, compaction requested=true
2024-11-09T21:25:31,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:31,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc0e2a2014f92d28c5b9a7c68250eadb:info, priority=-2147483648, current under compaction store size is 1
2024-11-09T21:25:31,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:31,197 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-09T21:25:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:31,197 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc0e2a2014f92d28c5b9a7c68250eadb 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-09T21:25:31,198 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 74500 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-09T21:25:31,198 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1541): cc0e2a2014f92d28c5b9a7c68250eadb/info is initiating minor compaction (all files)
2024-11-09T21:25:31,198 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc0e2a2014f92d28c5b9a7c68250eadb/info in TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:31,198 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/1e6eacdd04044f31a92f6ee33108d146, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/4971793ab1cd407f934763ad884dbf2b, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/dec70af8d06c4f3db17b46967cff5f40] into tmpdir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp, totalSize=72.8 K
2024-11-09T21:25:31,199 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e6eacdd04044f31a92f6ee33108d146, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731187516911
2024-11-09T21:25:31,199 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4971793ab1cd407f934763ad884dbf2b, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731187531139
2024-11-09T21:25:31,199 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting dec70af8d06c4f3db17b46967cff5f40, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731187531150
2024-11-09T21:25:31,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/38c938961a194da4851c7f62f1801dc7 is 1080, key is row0117/info:/1731187531176/Put/seqid=0
2024-11-09T21:25:31,206 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741860_1036 (size=17906)
2024-11-09T21:25:31,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741860_1036 (size=17906)
2024-11-09T21:25:31,208 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/38c938961a194da4851c7f62f1801dc7
2024-11-09T21:25:31,212 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc0e2a2014f92d28c5b9a7c68250eadb#info#compaction#75 average throughput is 28.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-09T21:25:31,213 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/f9352ef724c0420595638fe83f9bdd86 is 1080, key is row0062/info:/1731187516911/Put/seqid=0
2024-11-09T21:25:31,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/38c938961a194da4851c7f62f1801dc7 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/38c938961a194da4851c7f62f1801dc7
2024-11-09T21:25:31,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741861_1037 (size=64714)
2024-11-09T21:25:31,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741861_1037 (size=64714)
2024-11-09T21:25:31,218 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/38c938961a194da4851c7f62f1801dc7, entries=12, sequenceid=172, filesize=17.5 K
2024-11-09T21:25:31,219 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=0 B/0 for cc0e2a2014f92d28c5b9a7c68250eadb in 22ms, sequenceid=172, compaction requested=false
2024-11-09T21:25:31,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:31,222 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/f9352ef724c0420595638fe83f9bdd86 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/f9352ef724c0420595638fe83f9bdd86
2024-11-09T21:25:31,227 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc0e2a2014f92d28c5b9a7c68250eadb/info of cc0e2a2014f92d28c5b9a7c68250eadb into f9352ef724c0420595638fe83f9bdd86(size=63.2 K), total size for store is 80.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-09T21:25:31,227 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:31,227 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb., storeName=cc0e2a2014f92d28c5b9a7c68250eadb/info, priority=13, startTime=1731187531197; duration=0sec
2024-11-09T21:25:31,227 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:31,227 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc0e2a2014f92d28c5b9a7c68250eadb:info
2024-11-09T21:25:31,321 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:31,582 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:31,585 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:32,321 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:32,583 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:32,586 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:33,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:33,219 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc0e2a2014f92d28c5b9a7c68250eadb 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-09T21:25:33,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/ebbe4e61d0c04ed08377488952924db6 is 1080, key is row0129/info:/1731187533201/Put/seqid=0
2024-11-09T21:25:33,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741862_1038 (size=12516)
2024-11-09T21:25:33,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741862_1038 (size=12516)
2024-11-09T21:25:33,228 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/ebbe4e61d0c04ed08377488952924db6
2024-11-09T21:25:33,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/ebbe4e61d0c04ed08377488952924db6 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/ebbe4e61d0c04ed08377488952924db6
2024-11-09T21:25:33,239 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/ebbe4e61d0c04ed08377488952924db6, entries=7, sequenceid=183, filesize=12.2 K
2024-11-09T21:25:33,240 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9684 for cc0e2a2014f92d28c5b9a7c68250eadb in 21ms, sequenceid=183, compaction requested=true
2024-11-09T21:25:33,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:33,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc0e2a2014f92d28c5b9a7c68250eadb:info, priority=-2147483648, current under compaction store size is 1
2024-11-09T21:25:33,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:33,240 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-09T21:25:33,241 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 95136 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-09T21:25:33,241 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1541): cc0e2a2014f92d28c5b9a7c68250eadb/info is initiating minor compaction (all files)
2024-11-09T21:25:33,241 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc0e2a2014f92d28c5b9a7c68250eadb/info in TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:33,241 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/f9352ef724c0420595638fe83f9bdd86, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/38c938961a194da4851c7f62f1801dc7, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/ebbe4e61d0c04ed08377488952924db6] into tmpdir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp, totalSize=92.9 K
2024-11-09T21:25:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:33,241 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9352ef724c0420595638fe83f9bdd86, keycount=55, bloomtype=ROW, size=63.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731187516911
2024-11-09T21:25:33,241 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc0e2a2014f92d28c5b9a7c68250eadb 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-09T21:25:33,242 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38c938961a194da4851c7f62f1801dc7, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731187531176
2024-11-09T21:25:33,242 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebbe4e61d0c04ed08377488952924db6, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731187533201
2024-11-09T21:25:33,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/46f206061d59486dbad58c6b90f6e4e0 is 1080, key is row0136/info:/1731187533220/Put/seqid=0
2024-11-09T21:25:33,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741863_1039 (size=16828)
2024-11-09T21:25:33,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741863_1039 (size=16828)
2024-11-09T21:25:33,251 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/46f206061d59486dbad58c6b90f6e4e0
2024-11-09T21:25:33,254 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc0e2a2014f92d28c5b9a7c68250eadb#info#compaction#78 average throughput is 37.97 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-09T21:25:33,256 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/a6bda7b901f54a73a11d45439dc543f3 is 1080, key is row0062/info:/1731187516911/Put/seqid=0
2024-11-09T21:25:33,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/46f206061d59486dbad58c6b90f6e4e0 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/46f206061d59486dbad58c6b90f6e4e0
2024-11-09T21:25:33,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741864_1040 (size=85371)
2024-11-09T21:25:33,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741864_1040 (size=85371)
2024-11-09T21:25:33,265 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/46f206061d59486dbad58c6b90f6e4e0, entries=11, sequenceid=197, filesize=16.4 K
2024-11-09T21:25:33,266 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=9.46 KB/9684 for cc0e2a2014f92d28c5b9a7c68250eadb in 25ms, sequenceid=197, compaction requested=false
2024-11-09T21:25:33,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:33,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:33,268 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc0e2a2014f92d28c5b9a7c68250eadb 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-09T21:25:33,270 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/a6bda7b901f54a73a11d45439dc543f3 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/a6bda7b901f54a73a11d45439dc543f3
2024-11-09T21:25:33,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/6b5dee29fede4d0fafdb04b5cf055cbe is 1080, key is row0147/info:/1731187533243/Put/seqid=0
2024-11-09T21:25:33,277 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741865_1041 (size=16828)
2024-11-09T21:25:33,277 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc0e2a2014f92d28c5b9a7c68250eadb/info of cc0e2a2014f92d28c5b9a7c68250eadb into a6bda7b901f54a73a11d45439dc543f3(size=83.4 K), total size for store is 99.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-09T21:25:33,277 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:33,277 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb., storeName=cc0e2a2014f92d28c5b9a7c68250eadb/info, priority=13, startTime=1731187533240; duration=0sec
2024-11-09T21:25:33,277 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741865_1041 (size=16828)
2024-11-09T21:25:33,277 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:33,277 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc0e2a2014f92d28c5b9a7c68250eadb:info
2024-11-09T21:25:33,277 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/6b5dee29fede4d0fafdb04b5cf055cbe
2024-11-09T21:25:33,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/6b5dee29fede4d0fafdb04b5cf055cbe as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/6b5dee29fede4d0fafdb04b5cf055cbe
2024-11-09T21:25:33,286 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/6b5dee29fede4d0fafdb04b5cf055cbe, entries=11, sequenceid=211, filesize=16.4 K
2024-11-09T21:25:33,287 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=3.15 KB/3228 for cc0e2a2014f92d28c5b9a7c68250eadb in 19ms, sequenceid=211, compaction requested=true
2024-11-09T21:25:33,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:33,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc0e2a2014f92d28c5b9a7c68250eadb:info, priority=-2147483648, current under compaction store size is 1
2024-11-09T21:25:33,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:33,288 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-09T21:25:33,289 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 119027 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-09T21:25:33,289 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1541): cc0e2a2014f92d28c5b9a7c68250eadb/info is initiating minor compaction (all files)
2024-11-09T21:25:33,289 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc0e2a2014f92d28c5b9a7c68250eadb/info in TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:33,289 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/a6bda7b901f54a73a11d45439dc543f3, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/46f206061d59486dbad58c6b90f6e4e0, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/6b5dee29fede4d0fafdb04b5cf055cbe] into tmpdir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp, totalSize=116.2 K
2024-11-09T21:25:33,289 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6bda7b901f54a73a11d45439dc543f3, keycount=74, bloomtype=ROW, size=83.4 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731187516911
2024-11-09T21:25:33,290 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46f206061d59486dbad58c6b90f6e4e0, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731187533220
2024-11-09T21:25:33,290 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b5dee29fede4d0fafdb04b5cf055cbe, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731187533243
2024-11-09T21:25:33,301 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc0e2a2014f92d28c5b9a7c68250eadb#info#compaction#80 average throughput is 32.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-09T21:25:33,301 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/129ccc522e0345b8beac88ef429c7167 is 1080, key is row0062/info:/1731187516911/Put/seqid=0
2024-11-09T21:25:33,304 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741866_1042 (size=109197)
2024-11-09T21:25:33,305 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741866_1042 (size=109197)
2024-11-09T21:25:33,310 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/129ccc522e0345b8beac88ef429c7167 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/129ccc522e0345b8beac88ef429c7167
2024-11-09T21:25:33,316 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc0e2a2014f92d28c5b9a7c68250eadb/info of cc0e2a2014f92d28c5b9a7c68250eadb into 129ccc522e0345b8beac88ef429c7167(size=106.6 K), total size for store is 106.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-09T21:25:33,316 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:33,316 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb., storeName=cc0e2a2014f92d28c5b9a7c68250eadb/info, priority=13, startTime=1731187533287; duration=0sec
2024-11-09T21:25:33,316 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:33,316 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc0e2a2014f92d28c5b9a7c68250eadb:info
2024-11-09T21:25:33,322 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:33,410 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-09T21:25:33,585 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:33,587 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:34,322 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:34,586 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:34,588 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:35,285 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc0e2a2014f92d28c5b9a7c68250eadb 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-09T21:25:35,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/b52a625f4ea44be5b86a73b4c254b5f3 is 1080, key is row0158/info:/1731187533269/Put/seqid=0
2024-11-09T21:25:35,294 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741867_1043 (size=12516)
2024-11-09T21:25:35,294 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741867_1043 (size=12516)
2024-11-09T21:25:35,295 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/b52a625f4ea44be5b86a73b4c254b5f3
2024-11-09T21:25:35,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/b52a625f4ea44be5b86a73b4c254b5f3 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/b52a625f4ea44be5b86a73b4c254b5f3
2024-11-09T21:25:35,306 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/b52a625f4ea44be5b86a73b4c254b5f3, entries=7, sequenceid=223, filesize=12.2 K
2024-11-09T21:25:35,317 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for cc0e2a2014f92d28c5b9a7c68250eadb in 32ms, sequenceid=223, compaction requested=false
2024-11-09T21:25:35,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:35,318 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc0e2a2014f92d28c5b9a7c68250eadb 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-09T21:25:35,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/b8fc7d41c09b4d78a3e3ad49ecb9bf66 is 1080, key is row0165/info:/1731187535286/Put/seqid=0
2024-11-09T21:25:35,323 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:35,326 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741868_1044 (size=16828)
2024-11-09T21:25:35,327 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/b8fc7d41c09b4d78a3e3ad49ecb9bf66
2024-11-09T21:25:35,327 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741868_1044 (size=16828)
2024-11-09T21:25:35,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/b8fc7d41c09b4d78a3e3ad49ecb9bf66 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/b8fc7d41c09b4d78a3e3ad49ecb9bf66
2024-11-09T21:25:35,337 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/b8fc7d41c09b4d78a3e3ad49ecb9bf66, entries=11, sequenceid=237, filesize=16.4 K
2024-11-09T21:25:35,338 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=10.51 KB/10760 for cc0e2a2014f92d28c5b9a7c68250eadb in 20ms, sequenceid=237, compaction requested=true
2024-11-09T21:25:35,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:35,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc0e2a2014f92d28c5b9a7c68250eadb:info, priority=-2147483648, current under compaction store size is 1
2024-11-09T21:25:35,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:35,339 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-09T21:25:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:35,339 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc0e2a2014f92d28c5b9a7c68250eadb 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-09T21:25:35,340 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 138541 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-09T21:25:35,340 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1541): cc0e2a2014f92d28c5b9a7c68250eadb/info is initiating minor compaction (all files)
2024-11-09T21:25:35,340 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc0e2a2014f92d28c5b9a7c68250eadb/info in TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:35,340 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/129ccc522e0345b8beac88ef429c7167, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/b52a625f4ea44be5b86a73b4c254b5f3, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/b8fc7d41c09b4d78a3e3ad49ecb9bf66] into tmpdir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp, totalSize=135.3 K
2024-11-09T21:25:35,340 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 129ccc522e0345b8beac88ef429c7167, keycount=96, bloomtype=ROW, size=106.6 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731187516911
2024-11-09T21:25:35,340 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting b52a625f4ea44be5b86a73b4c254b5f3, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731187533269
2024-11-09T21:25:35,341 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8fc7d41c09b4d78a3e3ad49ecb9bf66, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731187535286
2024-11-09T21:25:35,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/95f914ca6d174f429add365f7072cb19 is 1080, key is row0176/info:/1731187535319/Put/seqid=0
2024-11-09T21:25:35,353 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741869_1045 (size=16828)
2024-11-09T21:25:35,353 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741869_1045 (size=16828)
2024-11-09T21:25:35,353 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/95f914ca6d174f429add365f7072cb19
2024-11-09T21:25:35,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/95f914ca6d174f429add365f7072cb19 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/95f914ca6d174f429add365f7072cb19
2024-11-09T21:25:35,359 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc0e2a2014f92d28c5b9a7c68250eadb#info#compaction#84 average throughput is 38.99 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-09T21:25:35,360 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/288fd74ead0e4c17919ac94e245ea5ae is 1080, key is row0062/info:/1731187516911/Put/seqid=0
2024-11-09T21:25:35,363 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741870_1046 (size=128835)
2024-11-09T21:25:35,364 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741870_1046 (size=128835)
2024-11-09T21:25:35,364 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/95f914ca6d174f429add365f7072cb19, entries=11, sequenceid=251, filesize=16.4 K
2024-11-09T21:25:35,365 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=6.30 KB/6456 for cc0e2a2014f92d28c5b9a7c68250eadb in 26ms, sequenceid=251, compaction requested=false
2024-11-09T21:25:35,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:35,370 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/288fd74ead0e4c17919ac94e245ea5ae as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/288fd74ead0e4c17919ac94e245ea5ae
2024-11-09T21:25:35,376 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc0e2a2014f92d28c5b9a7c68250eadb/info of cc0e2a2014f92d28c5b9a7c68250eadb into 288fd74ead0e4c17919ac94e245ea5ae(size=125.8 K), total size for store is 142.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-09T21:25:35,376 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:35,376 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb., storeName=cc0e2a2014f92d28c5b9a7c68250eadb/info, priority=13, startTime=1731187535338; duration=0sec
2024-11-09T21:25:35,376 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:35,376 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc0e2a2014f92d28c5b9a7c68250eadb:info
2024-11-09T21:25:35,587 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:35,589 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:36,323 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:36,587 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:36,589 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:37,324 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:37,359 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc0e2a2014f92d28c5b9a7c68250eadb 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-09T21:25:37,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/823d0ce0a71e43a39444c6987331f333 is 1080, key is row0187/info:/1731187535340/Put/seqid=0
2024-11-09T21:25:37,372 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741871_1047 (size=12520)
2024-11-09T21:25:37,372 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741871_1047 (size=12520)
2024-11-09T21:25:37,373 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/823d0ce0a71e43a39444c6987331f333
2024-11-09T21:25:37,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/823d0ce0a71e43a39444c6987331f333 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/823d0ce0a71e43a39444c6987331f333
2024-11-09T21:25:37,384 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/823d0ce0a71e43a39444c6987331f333, entries=7, sequenceid=262, filesize=12.2 K
2024-11-09T21:25:37,385 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for cc0e2a2014f92d28c5b9a7c68250eadb in 26ms, sequenceid=262, compaction requested=true
2024-11-09T21:25:37,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:37,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc0e2a2014f92d28c5b9a7c68250eadb:info, priority=-2147483648, current under compaction store size is 1
2024-11-09T21:25:37,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:37,385 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-09T21:25:37,386 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 158183 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-09T21:25:37,386 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1541): cc0e2a2014f92d28c5b9a7c68250eadb/info is initiating minor compaction (all files)
2024-11-09T21:25:37,386 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc0e2a2014f92d28c5b9a7c68250eadb/info in TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:37,386 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/288fd74ead0e4c17919ac94e245ea5ae, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/95f914ca6d174f429add365f7072cb19, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/823d0ce0a71e43a39444c6987331f333] into tmpdir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp, totalSize=154.5 K
2024-11-09T21:25:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:37,386 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc0e2a2014f92d28c5b9a7c68250eadb 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-09T21:25:37,387 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 288fd74ead0e4c17919ac94e245ea5ae, keycount=114, bloomtype=ROW, size=125.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731187516911
2024-11-09T21:25:37,387 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95f914ca6d174f429add365f7072cb19, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731187535319
2024-11-09T21:25:37,387 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 823d0ce0a71e43a39444c6987331f333, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731187535340
2024-11-09T21:25:37,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/1f49da0e261941cabdaeca8d4224b79f is 1080, key is row0194/info:/1731187537361/Put/seqid=0
2024-11-09T21:25:37,395 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741872_1048 (size=16839)
2024-11-09T21:25:37,395 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741872_1048 (size=16839)
2024-11-09T21:25:37,395 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/1f49da0e261941cabdaeca8d4224b79f
2024-11-09T21:25:37,399 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc0e2a2014f92d28c5b9a7c68250eadb#info#compaction#87 average throughput is 45.15 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-09T21:25:37,400 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/d267a9718c9c44aab010500b42c67ee7 is 1080, key is row0062/info:/1731187516911/Put/seqid=0
2024-11-09T21:25:37,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/1f49da0e261941cabdaeca8d4224b79f as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/1f49da0e261941cabdaeca8d4224b79f
2024-11-09T21:25:37,404 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741873_1049 (size=148418)
2024-11-09T21:25:37,404 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741873_1049 (size=148418)
2024-11-09T21:25:37,408 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/1f49da0e261941cabdaeca8d4224b79f, entries=11, sequenceid=276, filesize=16.4 K
2024-11-09T21:25:37,409 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=10.51 KB/10760 for cc0e2a2014f92d28c5b9a7c68250eadb in 23ms, sequenceid=276, compaction requested=false
2024-11-09T21:25:37,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:37,411 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/d267a9718c9c44aab010500b42c67ee7 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/d267a9718c9c44aab010500b42c67ee7
2024-11-09T21:25:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:37,411 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc0e2a2014f92d28c5b9a7c68250eadb 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-09T21:25:37,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/8c091f0cf5db44138ebc5b822b85d5a9 is 1080, key is row0205/info:/1731187537388/Put/seqid=0
2024-11-09T21:25:37,417 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc0e2a2014f92d28c5b9a7c68250eadb/info of cc0e2a2014f92d28c5b9a7c68250eadb into d267a9718c9c44aab010500b42c67ee7(size=144.9 K), total size for store is 161.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-09T21:25:37,418 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:37,418 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb., storeName=cc0e2a2014f92d28c5b9a7c68250eadb/info, priority=13, startTime=1731187537385; duration=0sec
2024-11-09T21:25:37,418 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:37,418 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc0e2a2014f92d28c5b9a7c68250eadb:info
2024-11-09T21:25:37,424 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741874_1050 (size=17918)
2024-11-09T21:25:37,424 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741874_1050 (size=17918)
2024-11-09T21:25:37,425 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/8c091f0cf5db44138ebc5b822b85d5a9
2024-11-09T21:25:37,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/8c091f0cf5db44138ebc5b822b85d5a9 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/8c091f0cf5db44138ebc5b822b85d5a9
2024-11-09T21:25:37,435 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/8c091f0cf5db44138ebc5b822b85d5a9, entries=12, sequenceid=291, filesize=17.5 K
2024-11-09T21:25:37,436 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=8.41 KB/8608 for cc0e2a2014f92d28c5b9a7c68250eadb in 25ms, sequenceid=291, compaction requested=true
2024-11-09T21:25:37,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:37,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc0e2a2014f92d28c5b9a7c68250eadb:info, priority=-2147483648, current under compaction store size is 1
2024-11-09T21:25:37,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:37,436 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-09T21:25:37,437 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 183175 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-09T21:25:37,437 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1541): cc0e2a2014f92d28c5b9a7c68250eadb/info is initiating minor compaction (all files)
2024-11-09T21:25:37,437 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc0e2a2014f92d28c5b9a7c68250eadb/info in TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:37,438 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/d267a9718c9c44aab010500b42c67ee7, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/1f49da0e261941cabdaeca8d4224b79f, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/8c091f0cf5db44138ebc5b822b85d5a9] into tmpdir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp, totalSize=178.9 K
2024-11-09T21:25:37,438 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting d267a9718c9c44aab010500b42c67ee7, keycount=132, bloomtype=ROW, size=144.9 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731187516911
2024-11-09T21:25:37,438 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f49da0e261941cabdaeca8d4224b79f, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731187537361
2024-11-09T21:25:37,438 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c091f0cf5db44138ebc5b822b85d5a9, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731187537388
2024-11-09T21:25:37,449 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc0e2a2014f92d28c5b9a7c68250eadb#info#compaction#89 average throughput is 53.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-09T21:25:37,450 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/da0ecb8adbdf438b85e306de4dbca59f is 1080, key is row0062/info:/1731187516911/Put/seqid=0
2024-11-09T21:25:37,452 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741875_1051 (size=173345)
2024-11-09T21:25:37,452 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741875_1051 (size=173345)
2024-11-09T21:25:37,458 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/da0ecb8adbdf438b85e306de4dbca59f as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/da0ecb8adbdf438b85e306de4dbca59f
2024-11-09T21:25:37,464 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc0e2a2014f92d28c5b9a7c68250eadb/info of cc0e2a2014f92d28c5b9a7c68250eadb into da0ecb8adbdf438b85e306de4dbca59f(size=169.3 K), total size for store is 169.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-09T21:25:37,464 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:37,464 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb., storeName=cc0e2a2014f92d28c5b9a7c68250eadb/info, priority=13, startTime=1731187537436; duration=0sec
2024-11-09T21:25:37,464 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:37,464 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc0e2a2014f92d28c5b9a7c68250eadb:info
2024-11-09T21:25:37,588 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:37,590 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:38,325 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:38,589 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:38,590 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:39,326 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:39,433 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc0e2a2014f92d28c5b9a7c68250eadb 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-09T21:25:39,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/0f554f4b2a8c452cafb5fcd4738f419b is 1080, key is row0217/info:/1731187537412/Put/seqid=0
2024-11-09T21:25:39,445 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741876_1052 (size=14681)
2024-11-09T21:25:39,445 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741876_1052 (size=14681)
2024-11-09T21:25:39,446 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/0f554f4b2a8c452cafb5fcd4738f419b
2024-11-09T21:25:39,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/0f554f4b2a8c452cafb5fcd4738f419b as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/0f554f4b2a8c452cafb5fcd4738f419b
2024-11-09T21:25:39,458 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/0f554f4b2a8c452cafb5fcd4738f419b, entries=9, sequenceid=305, filesize=14.3 K
2024-11-09T21:25:39,459 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=10.51 KB/10760 for cc0e2a2014f92d28c5b9a7c68250eadb in 27ms, sequenceid=305, compaction requested=false
2024-11-09T21:25:39,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:39,461 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc0e2a2014f92d28c5b9a7c68250eadb 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-09T21:25:39,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/5809a15ff38b465e8453a4b13c2cbe8b is 1080, key is row0226/info:/1731187539435/Put/seqid=0
2024-11-09T21:25:39,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741877_1053 (size=16839)
2024-11-09T21:25:39,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741877_1053 (size=16839)
2024-11-09T21:25:39,470 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/5809a15ff38b465e8453a4b13c2cbe8b
2024-11-09T21:25:39,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/5809a15ff38b465e8453a4b13c2cbe8b as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/5809a15ff38b465e8453a4b13c2cbe8b
2024-11-09T21:25:39,481 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/5809a15ff38b465e8453a4b13c2cbe8b, entries=11, sequenceid=319, filesize=16.4 K
2024-11-09T21:25:39,482 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=9.46 KB/9684 for cc0e2a2014f92d28c5b9a7c68250eadb in 21ms, sequenceid=319, compaction requested=true
2024-11-09T21:25:39,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:39,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc0e2a2014f92d28c5b9a7c68250eadb:info, priority=-2147483648, current under compaction store size is 1
2024-11-09T21:25:39,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(8855): Flush requested on cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:39,482 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-09T21:25:39,482 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc0e2a2014f92d28c5b9a7c68250eadb 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-09T21:25:39,483 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 204865 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-09T21:25:39,483 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1541): cc0e2a2014f92d28c5b9a7c68250eadb/info is initiating minor compaction (all files)
2024-11-09T21:25:39,483 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc0e2a2014f92d28c5b9a7c68250eadb/info in TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:39,484 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/da0ecb8adbdf438b85e306de4dbca59f, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/0f554f4b2a8c452cafb5fcd4738f419b, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/5809a15ff38b465e8453a4b13c2cbe8b] into tmpdir=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp, totalSize=200.1 K
2024-11-09T21:25:39,484 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting da0ecb8adbdf438b85e306de4dbca59f, keycount=155, bloomtype=ROW, size=169.3 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731187516911
2024-11-09T21:25:39,484 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f554f4b2a8c452cafb5fcd4738f419b, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731187537412
2024-11-09T21:25:39,485 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5809a15ff38b465e8453a4b13c2cbe8b, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731187539435
2024-11-09T21:25:39,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/a1799a87274f4d2996cf9de46e6eaf70 is 1080, key is row0237/info:/1731187539462/Put/seqid=0
2024-11-09T21:25:39,496 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741878_1054 (size=15760)
2024-11-09T21:25:39,497 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741878_1054 (size=15760)
2024-11-09T21:25:39,498 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/a1799a87274f4d2996cf9de46e6eaf70
2024-11-09T21:25:39,500 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc0e2a2014f92d28c5b9a7c68250eadb#info#compaction#93 average throughput is 44.89 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-09T21:25:39,501 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/655c5e40462a4a3f81712a9317b4c0be is 1080, key is row0062/info:/1731187516911/Put/seqid=0
2024-11-09T21:25:39,503 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741879_1055 (size=195031)
2024-11-09T21:25:39,503 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741879_1055 (size=195031)
2024-11-09T21:25:39,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/a1799a87274f4d2996cf9de46e6eaf70 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/a1799a87274f4d2996cf9de46e6eaf70
2024-11-09T21:25:39,509 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/655c5e40462a4a3f81712a9317b4c0be as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/655c5e40462a4a3f81712a9317b4c0be
2024-11-09T21:25:39,509 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/a1799a87274f4d2996cf9de46e6eaf70, entries=10, sequenceid=332, filesize=15.4 K
2024-11-09T21:25:39,510 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=10.51 KB/10760 for cc0e2a2014f92d28c5b9a7c68250eadb in 28ms, sequenceid=332, compaction requested=false
2024-11-09T21:25:39,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:39,515 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc0e2a2014f92d28c5b9a7c68250eadb/info of cc0e2a2014f92d28c5b9a7c68250eadb into 655c5e40462a4a3f81712a9317b4c0be(size=190.5 K), total size for store is 205.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-09T21:25:39,515 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:39,515 INFO  [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb., storeName=cc0e2a2014f92d28c5b9a7c68250eadb/info, priority=13, startTime=1731187539482; duration=0sec
2024-11-09T21:25:39,516 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-09T21:25:39,516 DEBUG [RS:0;d86ff69d6719:34269-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc0e2a2014f92d28c5b9a7c68250eadb:info
2024-11-09T21:25:39,590 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:39,591 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:40,327 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:40,590 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:40,592 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:41,328 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:41,506 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-11-09T21:25:41,507 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C34269%2C1731187503585.1731187541506
2024-11-09T21:25:41,511 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,511 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,512 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,512 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,512 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,512 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/WALs/d86ff69d6719,34269,1731187503585/d86ff69d6719%2C34269%2C1731187503585.1731187504098 with entries=320, filesize=311.00 KB; new WAL /user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/WALs/d86ff69d6719,34269,1731187503585/d86ff69d6719%2C34269%2C1731187503585.1731187541506
2024-11-09T21:25:41,515 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42767:42767),(127.0.0.1/127.0.0.1:43417:43417)]
2024-11-09T21:25:41,515 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/WALs/d86ff69d6719,34269,1731187503585/d86ff69d6719%2C34269%2C1731187503585.1731187504098 is not closed yet, will try archiving it next time
2024-11-09T21:25:41,516 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741833_1009 (size=318475)
2024-11-09T21:25:41,516 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741833_1009 (size=318475)
2024-11-09T21:25:41,520 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing cc0e2a2014f92d28c5b9a7c68250eadb 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-09T21:25:41,525 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/cf33b9bc1cd548b9aaca1c599dbe1c6a is 1080, key is row0247/info:/1731187539484/Put/seqid=0
2024-11-09T21:25:41,532 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741881_1057 (size=15760)
2024-11-09T21:25:41,532 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741881_1057 (size=15760)
2024-11-09T21:25:41,532 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/cf33b9bc1cd548b9aaca1c599dbe1c6a
2024-11-09T21:25:41,537 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/.tmp/info/cf33b9bc1cd548b9aaca1c599dbe1c6a as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/cf33b9bc1cd548b9aaca1c599dbe1c6a
2024-11-09T21:25:41,541 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/cf33b9bc1cd548b9aaca1c599dbe1c6a, entries=10, sequenceid=346, filesize=15.4 K
2024-11-09T21:25:41,542 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=0 B/0 for cc0e2a2014f92d28c5b9a7c68250eadb in 22ms, sequenceid=346, compaction requested=true
2024-11-09T21:25:41,542 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for cc0e2a2014f92d28c5b9a7c68250eadb:

2024-11-09T21:25:41,542 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB
2024-11-09T21:25:41,546 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/.tmp/info/e530176728e14f02b67ef05f99440b0c is 193, key is TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb./info:regioninfo/1731187519712/Put/seqid=0
2024-11-09T21:25:41,550 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741882_1058 (size=6223)
2024-11-09T21:25:41,550 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741882_1058 (size=6223)
2024-11-09T21:25:41,550 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/.tmp/info/e530176728e14f02b67ef05f99440b0c
2024-11-09T21:25:41,555 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/.tmp/info/e530176728e14f02b67ef05f99440b0c as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/info/e530176728e14f02b67ef05f99440b0c
2024-11-09T21:25:41,559 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/info/e530176728e14f02b67ef05f99440b0c, entries=5, sequenceid=21, filesize=6.1 K
2024-11-09T21:25:41,560 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 18ms, sequenceid=21, compaction requested=false
2024-11-09T21:25:41,560 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-09T21:25:41,560 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f16719376f7f3978f93657892e985b13:

2024-11-09T21:25:41,560 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C34269%2C1731187503585.1731187541560
2024-11-09T21:25:41,564 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,565 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,565 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,565 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,565 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,565 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/WALs/d86ff69d6719,34269,1731187503585/d86ff69d6719%2C34269%2C1731187503585.1731187541506 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/WALs/d86ff69d6719,34269,1731187503585/d86ff69d6719%2C34269%2C1731187503585.1731187541560
2024-11-09T21:25:41,566 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741880_1056 (size=731)
2024-11-09T21:25:41,566 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741880_1056 (size=731)
2024-11-09T21:25:41,573 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/WALs/d86ff69d6719,34269,1731187503585/d86ff69d6719%2C34269%2C1731187503585.1731187504098 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/oldWALs/d86ff69d6719%2C34269%2C1731187503585.1731187504098
2024-11-09T21:25:41,573 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43417:43417),(127.0.0.1/127.0.0.1:42767:42767)]
2024-11-09T21:25:41,574 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-11-09T21:25:41,574 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-09T21:25:41,574 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-09T21:25:41,574 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:25:41,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:41,574 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/WALs/d86ff69d6719,34269,1731187503585/d86ff69d6719%2C34269%2C1731187503585.1731187541506 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/oldWALs/d86ff69d6719%2C34269%2C1731187503585.1731187541506
2024-11-09T21:25:41,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:41,574 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-09T21:25:41,575 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-09T21:25:41,575 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1659811253, stopped=false
2024-11-09T21:25:41,575 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d86ff69d6719,46193,1731187503426
2024-11-09T21:25:41,591 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:41,592 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:41,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-09T21:25:41,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-09T21:25:41,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:41,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:41,627 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-09T21:25:41,627 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-09T21:25:41,628 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:25:41,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:41,628 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd86ff69d6719,34269,1731187503585' *****
2024-11-09T21:25:41,628 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:25:41,628 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:25:41,628 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-09T21:25:41,628 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-09T21:25:41,628 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-09T21:25:41,628 INFO  [RS:0;d86ff69d6719:34269 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-09T21:25:41,628 INFO  [RS:0;d86ff69d6719:34269 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-09T21:25:41,629 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(3091): Received CLOSE for cc0e2a2014f92d28c5b9a7c68250eadb
2024-11-09T21:25:41,629 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(3091): Received CLOSE for f16719376f7f3978f93657892e985b13
2024-11-09T21:25:41,629 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(959): stopping server d86ff69d6719,34269,1731187503585
2024-11-09T21:25:41,629 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-09T21:25:41,629 INFO  [RS:0;d86ff69d6719:34269 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d86ff69d6719:34269.
2024-11-09T21:25:41,629 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cc0e2a2014f92d28c5b9a7c68250eadb, disabling compactions & flushes
2024-11-09T21:25:41,629 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:41,629 DEBUG [RS:0;d86ff69d6719:34269 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:25:41,629 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:41,629 DEBUG [RS:0;d86ff69d6719:34269 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:41,629 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb. after waiting 0 ms
2024-11-09T21:25:41,629 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:41,629 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-09T21:25:41,629 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-09T21:25:41,629 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-09T21:25:41,629 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-09T21:25:41,629 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-11-09T21:25:41,629 DEBUG [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(1325): Online Regions={cc0e2a2014f92d28c5b9a7c68250eadb=TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb., 1588230740=hbase:meta,,1.1588230740, f16719376f7f3978f93657892e985b13=TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.}
2024-11-09T21:25:41,629 DEBUG [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, cc0e2a2014f92d28c5b9a7c68250eadb, f16719376f7f3978f93657892e985b13
2024-11-09T21:25:41,629 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-09T21:25:41,629 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-09T21:25:41,629 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-09T21:25:41,629 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-09T21:25:41,629 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-09T21:25:41,629 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/c3b2d5b3d4644ccbaac48861a5dfee70.d3a6c7157ea5ef8d32109ac560fabda4->hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/c3b2d5b3d4644ccbaac48861a5dfee70-top, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/TestLogRolling-testLogRolling=d3a6c7157ea5ef8d32109ac560fabda4-20b6b0208a234b759908b05ee0a3c18e, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/1e6eacdd04044f31a92f6ee33108d146, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/TestLogRolling-testLogRolling=d3a6c7157ea5ef8d32109ac560fabda4-c3540fd1cbd44748a1ac950ed56daac0, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/4971793ab1cd407f934763ad884dbf2b, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/f9352ef724c0420595638fe83f9bdd86, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/dec70af8d06c4f3db17b46967cff5f40, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/38c938961a194da4851c7f62f1801dc7, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/a6bda7b901f54a73a11d45439dc543f3, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/ebbe4e61d0c04ed08377488952924db6, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/46f206061d59486dbad58c6b90f6e4e0, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/129ccc522e0345b8beac88ef429c7167, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/6b5dee29fede4d0fafdb04b5cf055cbe, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/b52a625f4ea44be5b86a73b4c254b5f3, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/288fd74ead0e4c17919ac94e245ea5ae, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/b8fc7d41c09b4d78a3e3ad49ecb9bf66, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/95f914ca6d174f429add365f7072cb19, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/d267a9718c9c44aab010500b42c67ee7, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/823d0ce0a71e43a39444c6987331f333, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/1f49da0e261941cabdaeca8d4224b79f, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/da0ecb8adbdf438b85e306de4dbca59f, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/8c091f0cf5db44138ebc5b822b85d5a9, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/0f554f4b2a8c452cafb5fcd4738f419b, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/5809a15ff38b465e8453a4b13c2cbe8b] to archive
2024-11-09T21:25:41,630 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-09T21:25:41,632 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/c3b2d5b3d4644ccbaac48861a5dfee70.d3a6c7157ea5ef8d32109ac560fabda4 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/c3b2d5b3d4644ccbaac48861a5dfee70.d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:41,633 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/TestLogRolling-testLogRolling=d3a6c7157ea5ef8d32109ac560fabda4-20b6b0208a234b759908b05ee0a3c18e to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/TestLogRolling-testLogRolling=d3a6c7157ea5ef8d32109ac560fabda4-20b6b0208a234b759908b05ee0a3c18e
2024-11-09T21:25:41,634 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/1e6eacdd04044f31a92f6ee33108d146 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/1e6eacdd04044f31a92f6ee33108d146
2024-11-09T21:25:41,635 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-11-09T21:25:41,635 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/TestLogRolling-testLogRolling=d3a6c7157ea5ef8d32109ac560fabda4-c3540fd1cbd44748a1ac950ed56daac0 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/TestLogRolling-testLogRolling=d3a6c7157ea5ef8d32109ac560fabda4-c3540fd1cbd44748a1ac950ed56daac0
2024-11-09T21:25:41,636 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-09T21:25:41,636 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-09T21:25:41,636 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731187541629Running coprocessor pre-close hooks at 1731187541629Disabling compacts and flushes for region at 1731187541629Disabling writes for close at 1731187541629Writing region close event to WAL at 1731187541632 (+3 ms)Running coprocessor post-close hooks at 1731187541635 (+3 ms)Closed at 1731187541636 (+1 ms)
2024-11-09T21:25:41,636 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-09T21:25:41,636 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/4971793ab1cd407f934763ad884dbf2b to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/4971793ab1cd407f934763ad884dbf2b
2024-11-09T21:25:41,637 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/f9352ef724c0420595638fe83f9bdd86 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/f9352ef724c0420595638fe83f9bdd86
2024-11-09T21:25:41,638 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/dec70af8d06c4f3db17b46967cff5f40 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/dec70af8d06c4f3db17b46967cff5f40
2024-11-09T21:25:41,639 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/38c938961a194da4851c7f62f1801dc7 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/38c938961a194da4851c7f62f1801dc7
2024-11-09T21:25:41,640 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/a6bda7b901f54a73a11d45439dc543f3 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/a6bda7b901f54a73a11d45439dc543f3
2024-11-09T21:25:41,641 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/ebbe4e61d0c04ed08377488952924db6 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/ebbe4e61d0c04ed08377488952924db6
2024-11-09T21:25:41,642 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/46f206061d59486dbad58c6b90f6e4e0 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/46f206061d59486dbad58c6b90f6e4e0
2024-11-09T21:25:41,643 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/129ccc522e0345b8beac88ef429c7167 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/129ccc522e0345b8beac88ef429c7167
2024-11-09T21:25:41,645 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/6b5dee29fede4d0fafdb04b5cf055cbe to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/6b5dee29fede4d0fafdb04b5cf055cbe
2024-11-09T21:25:41,645 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/b52a625f4ea44be5b86a73b4c254b5f3 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/b52a625f4ea44be5b86a73b4c254b5f3
2024-11-09T21:25:41,647 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/288fd74ead0e4c17919ac94e245ea5ae to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/288fd74ead0e4c17919ac94e245ea5ae
2024-11-09T21:25:41,648 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/b8fc7d41c09b4d78a3e3ad49ecb9bf66 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/b8fc7d41c09b4d78a3e3ad49ecb9bf66
2024-11-09T21:25:41,649 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/95f914ca6d174f429add365f7072cb19 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/95f914ca6d174f429add365f7072cb19
2024-11-09T21:25:41,649 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/d267a9718c9c44aab010500b42c67ee7 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/d267a9718c9c44aab010500b42c67ee7
2024-11-09T21:25:41,650 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/823d0ce0a71e43a39444c6987331f333 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/823d0ce0a71e43a39444c6987331f333
2024-11-09T21:25:41,652 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/1f49da0e261941cabdaeca8d4224b79f to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/1f49da0e261941cabdaeca8d4224b79f
2024-11-09T21:25:41,653 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/da0ecb8adbdf438b85e306de4dbca59f to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/da0ecb8adbdf438b85e306de4dbca59f
2024-11-09T21:25:41,653 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/8c091f0cf5db44138ebc5b822b85d5a9 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/8c091f0cf5db44138ebc5b822b85d5a9
2024-11-09T21:25:41,654 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/0f554f4b2a8c452cafb5fcd4738f419b to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/0f554f4b2a8c452cafb5fcd4738f419b
2024-11-09T21:25:41,655 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/5809a15ff38b465e8453a4b13c2cbe8b to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/info/5809a15ff38b465e8453a4b13c2cbe8b
2024-11-09T21:25:41,656 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d86ff69d6719:46193 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-09T21:25:41,656 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [1e6eacdd04044f31a92f6ee33108d146=42984, 4971793ab1cd407f934763ad884dbf2b=12516, f9352ef724c0420595638fe83f9bdd86=64714, dec70af8d06c4f3db17b46967cff5f40=19000, 38c938961a194da4851c7f62f1801dc7=17906, a6bda7b901f54a73a11d45439dc543f3=85371, ebbe4e61d0c04ed08377488952924db6=12516, 46f206061d59486dbad58c6b90f6e4e0=16828, 129ccc522e0345b8beac88ef429c7167=109197, 6b5dee29fede4d0fafdb04b5cf055cbe=16828, b52a625f4ea44be5b86a73b4c254b5f3=12516, 288fd74ead0e4c17919ac94e245ea5ae=128835, b8fc7d41c09b4d78a3e3ad49ecb9bf66=16828, 95f914ca6d174f429add365f7072cb19=16828, d267a9718c9c44aab010500b42c67ee7=148418, 823d0ce0a71e43a39444c6987331f333=12520, 1f49da0e261941cabdaeca8d4224b79f=16839, da0ecb8adbdf438b85e306de4dbca59f=173345, 8c091f0cf5db44138ebc5b822b85d5a9=17918, 0f554f4b2a8c452cafb5fcd4738f419b=14681, 5809a15ff38b465e8453a4b13c2cbe8b=16839]
2024-11-09T21:25:41,660 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/cc0e2a2014f92d28c5b9a7c68250eadb/recovered.edits/349.seqid, newMaxSeqId=349, maxSeqId=130
2024-11-09T21:25:41,660 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:41,660 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cc0e2a2014f92d28c5b9a7c68250eadb:
Waiting for close lock at 1731187541629Running coprocessor pre-close hooks at 1731187541629Disabling compacts and flushes for region at 1731187541629Disabling writes for close at 1731187541629Writing region close event to WAL at 1731187541656 (+27 ms)Running coprocessor post-close hooks at 1731187541660 (+4 ms)Closed at 1731187541660
2024-11-09T21:25:41,660 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1731187519024.cc0e2a2014f92d28c5b9a7c68250eadb.
2024-11-09T21:25:41,661 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f16719376f7f3978f93657892e985b13, disabling compactions & flushes
2024-11-09T21:25:41,661 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.
2024-11-09T21:25:41,661 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.
2024-11-09T21:25:41,661 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13. after waiting 0 ms
2024-11-09T21:25:41,661 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.
2024-11-09T21:25:41,661 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/f16719376f7f3978f93657892e985b13/info/c3b2d5b3d4644ccbaac48861a5dfee70.d3a6c7157ea5ef8d32109ac560fabda4->hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/d3a6c7157ea5ef8d32109ac560fabda4/info/c3b2d5b3d4644ccbaac48861a5dfee70-bottom] to archive
2024-11-09T21:25:41,662 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-09T21:25:41,664 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/f16719376f7f3978f93657892e985b13/info/c3b2d5b3d4644ccbaac48861a5dfee70.d3a6c7157ea5ef8d32109ac560fabda4 to hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/archive/data/default/TestLogRolling-testLogRolling/f16719376f7f3978f93657892e985b13/info/c3b2d5b3d4644ccbaac48861a5dfee70.d3a6c7157ea5ef8d32109ac560fabda4
2024-11-09T21:25:41,664 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-11-09T21:25:41,667 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/data/default/TestLogRolling-testLogRolling/f16719376f7f3978f93657892e985b13/recovered.edits/135.seqid, newMaxSeqId=135, maxSeqId=130
2024-11-09T21:25:41,668 INFO  [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.
2024-11-09T21:25:41,668 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f16719376f7f3978f93657892e985b13:
Waiting for close lock at 1731187541661Running coprocessor pre-close hooks at 1731187541661Disabling compacts and flushes for region at 1731187541661Disabling writes for close at 1731187541661Writing region close event to WAL at 1731187541664 (+3 ms)Running coprocessor post-close hooks at 1731187541668 (+4 ms)Closed at 1731187541668
2024-11-09T21:25:41,668 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1731187519024.f16719376f7f3978f93657892e985b13.
2024-11-09T21:25:41,829 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(976): stopping server d86ff69d6719,34269,1731187503585; all regions closed.
2024-11-09T21:25:41,830 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,831 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,831 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,831 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,831 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,835 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741834_1010 (size=8107)
2024-11-09T21:25:41,837 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741834_1010 (size=8107)
2024-11-09T21:25:41,841 DEBUG [RS:0;d86ff69d6719:34269 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/oldWALs
2024-11-09T21:25:41,841 INFO  [RS:0;d86ff69d6719:34269 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d86ff69d6719%2C34269%2C1731187503585.meta:.meta(num 1731187504544)
2024-11-09T21:25:41,842 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,842 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,842 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,843 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,843 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:41,845 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741883_1059 (size=780)
2024-11-09T21:25:41,845 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741883_1059 (size=780)
2024-11-09T21:25:41,847 DEBUG [RS:0;d86ff69d6719:34269 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/oldWALs
2024-11-09T21:25:41,847 INFO  [RS:0;d86ff69d6719:34269 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d86ff69d6719%2C34269%2C1731187503585:(num 1731187541560)
2024-11-09T21:25:41,847 DEBUG [RS:0;d86ff69d6719:34269 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:41,847 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.LeaseManager(133): Closed leases
2024-11-09T21:25:41,847 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-09T21:25:41,847 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.ChoreService(370): Chore service for: regionserver/d86ff69d6719:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-09T21:25:41,847 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-09T21:25:41,847 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-09T21:25:41,847 INFO  [RS:0;d86ff69d6719:34269 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34269
2024-11-09T21:25:41,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T21:25:41,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d86ff69d6719,34269,1731187503585
2024-11-09T21:25:41,869 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-09T21:25:41,869 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d86ff69d6719,34269,1731187503585]
2024-11-09T21:25:41,885 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d86ff69d6719,34269,1731187503585 already deleted, retry=false
2024-11-09T21:25:41,886 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d86ff69d6719,34269,1731187503585 expired; onlineServers=0
2024-11-09T21:25:41,886 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd86ff69d6719,46193,1731187503426' *****
2024-11-09T21:25:41,886 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-09T21:25:41,886 INFO  [M:0;d86ff69d6719:46193 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-09T21:25:41,886 INFO  [M:0;d86ff69d6719:46193 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-09T21:25:41,886 DEBUG [M:0;d86ff69d6719:46193 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-09T21:25:41,886 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-09T21:25:41,886 DEBUG [M:0;d86ff69d6719:46193 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-09T21:25:41,887 DEBUG [master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187503897 {}] cleaner.HFileCleaner(306): Exit Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187503897,5,FailOnTimeoutGroup]
2024-11-09T21:25:41,887 DEBUG [master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187503897 {}] cleaner.HFileCleaner(306): Exit Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187503897,5,FailOnTimeoutGroup]
2024-11-09T21:25:41,887 INFO  [M:0;d86ff69d6719:46193 {}] hbase.ChoreService(370): Chore service for: master/d86ff69d6719:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-09T21:25:41,887 INFO  [M:0;d86ff69d6719:46193 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-09T21:25:41,887 DEBUG [M:0;d86ff69d6719:46193 {}] master.HMaster(1795): Stopping service threads
2024-11-09T21:25:41,887 INFO  [M:0;d86ff69d6719:46193 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-09T21:25:41,887 INFO  [M:0;d86ff69d6719:46193 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-09T21:25:41,888 INFO  [M:0;d86ff69d6719:46193 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-09T21:25:41,888 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-09T21:25:41,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-09T21:25:41,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:41,894 DEBUG [M:0;d86ff69d6719:46193 {}] zookeeper.ZKUtil(347): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-09T21:25:41,894 WARN  [M:0;d86ff69d6719:46193 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-09T21:25:41,895 INFO  [M:0;d86ff69d6719:46193 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/.lastflushedseqids
2024-11-09T21:25:41,900 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741884_1060 (size=228)
2024-11-09T21:25:41,900 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741884_1060 (size=228)
2024-11-09T21:25:41,901 INFO  [M:0;d86ff69d6719:46193 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-09T21:25:41,901 INFO  [M:0;d86ff69d6719:46193 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-09T21:25:41,901 DEBUG [M:0;d86ff69d6719:46193 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-09T21:25:41,901 INFO  [M:0;d86ff69d6719:46193 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:41,901 DEBUG [M:0;d86ff69d6719:46193 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:41,901 DEBUG [M:0;d86ff69d6719:46193 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-09T21:25:41,901 DEBUG [M:0;d86ff69d6719:46193 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:41,901 INFO  [M:0;d86ff69d6719:46193 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.44 KB heapSize=63.39 KB
2024-11-09T21:25:41,919 DEBUG [M:0;d86ff69d6719:46193 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/47dfcb42f9104c2286d44d2896427987 is 82, key is hbase:meta,,1/info:regioninfo/1731187504568/Put/seqid=0
2024-11-09T21:25:41,923 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741885_1061 (size=5672)
2024-11-09T21:25:41,924 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741885_1061 (size=5672)
2024-11-09T21:25:41,924 INFO  [M:0;d86ff69d6719:46193 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/47dfcb42f9104c2286d44d2896427987
2024-11-09T21:25:41,943 DEBUG [M:0;d86ff69d6719:46193 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6398b6122404426ebb0c8236325db6dc is 751, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731187505112/Put/seqid=0
2024-11-09T21:25:41,947 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741886_1062 (size=7091)
2024-11-09T21:25:41,947 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741886_1062 (size=7091)
2024-11-09T21:25:41,948 INFO  [M:0;d86ff69d6719:46193 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.84 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6398b6122404426ebb0c8236325db6dc
2024-11-09T21:25:41,951 INFO  [M:0;d86ff69d6719:46193 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6398b6122404426ebb0c8236325db6dc
2024-11-09T21:25:41,965 DEBUG [M:0;d86ff69d6719:46193 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ebbd007aadba462283089206539fe359 is 69, key is d86ff69d6719,34269,1731187503585/rs:state/1731187503946/Put/seqid=0
2024-11-09T21:25:41,968 INFO  [regionserver/d86ff69d6719:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-09T21:25:41,969 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741887_1063 (size=5156)
2024-11-09T21:25:41,969 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741887_1063 (size=5156)
2024-11-09T21:25:41,970 INFO  [M:0;d86ff69d6719:46193 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ebbd007aadba462283089206539fe359
2024-11-09T21:25:41,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:25:41,977 INFO  [RS:0;d86ff69d6719:34269 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-09T21:25:41,977 INFO  [RS:0;d86ff69d6719:34269 {}] regionserver.HRegionServer(1031): Exiting; stopping=d86ff69d6719,34269,1731187503585; zookeeper connection closed.
2024-11-09T21:25:41,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x10122c683f90001, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:25:41,978 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@f3b4a9e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@f3b4a9e
2024-11-09T21:25:41,978 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-09T21:25:41,992 DEBUG [M:0;d86ff69d6719:46193 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0f1ab36548cb448e8d24b1a76bbf088e is 52, key is load_balancer_on/state:d/1731187504725/Put/seqid=0
2024-11-09T21:25:41,996 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741888_1064 (size=5056)
2024-11-09T21:25:41,996 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741888_1064 (size=5056)
2024-11-09T21:25:41,996 INFO  [M:0;d86ff69d6719:46193 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0f1ab36548cb448e8d24b1a76bbf088e
2024-11-09T21:25:42,001 DEBUG [M:0;d86ff69d6719:46193 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/47dfcb42f9104c2286d44d2896427987 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/47dfcb42f9104c2286d44d2896427987
2024-11-09T21:25:42,005 INFO  [M:0;d86ff69d6719:46193 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/47dfcb42f9104c2286d44d2896427987, entries=8, sequenceid=125, filesize=5.5 K
2024-11-09T21:25:42,006 DEBUG [M:0;d86ff69d6719:46193 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6398b6122404426ebb0c8236325db6dc as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6398b6122404426ebb0c8236325db6dc
2024-11-09T21:25:42,010 INFO  [M:0;d86ff69d6719:46193 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6398b6122404426ebb0c8236325db6dc
2024-11-09T21:25:42,011 INFO  [M:0;d86ff69d6719:46193 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6398b6122404426ebb0c8236325db6dc, entries=13, sequenceid=125, filesize=6.9 K
2024-11-09T21:25:42,011 DEBUG [M:0;d86ff69d6719:46193 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ebbd007aadba462283089206539fe359 as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ebbd007aadba462283089206539fe359
2024-11-09T21:25:42,016 INFO  [M:0;d86ff69d6719:46193 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ebbd007aadba462283089206539fe359, entries=1, sequenceid=125, filesize=5.0 K
2024-11-09T21:25:42,017 DEBUG [M:0;d86ff69d6719:46193 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0f1ab36548cb448e8d24b1a76bbf088e as hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0f1ab36548cb448e8d24b1a76bbf088e
2024-11-09T21:25:42,021 INFO  [M:0;d86ff69d6719:46193 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38249/user/jenkins/test-data/e8b633b9-9810-fad7-e58b-dd57fc3dab11/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0f1ab36548cb448e8d24b1a76bbf088e, entries=1, sequenceid=125, filesize=4.9 K
2024-11-09T21:25:42,023 INFO  [M:0;d86ff69d6719:46193 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.44 KB/52675, heapSize ~63.33 KB/64848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 121ms, sequenceid=125, compaction requested=false
2024-11-09T21:25:42,024 INFO  [M:0;d86ff69d6719:46193 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:42,024 DEBUG [M:0;d86ff69d6719:46193 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731187541901Disabling compacts and flushes for region at 1731187541901Disabling writes for close at 1731187541901Obtaining lock to block concurrent updates at 1731187541901Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731187541901Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52675, getHeapSize=64848, getOffHeapSize=0, getCellsCount=148 at 1731187541901Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731187541902 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731187541902Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731187541919 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731187541919Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731187541928 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731187541943 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731187541943Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731187541951 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731187541964 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731187541964Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731187541974 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731187541991 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731187541991Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4825fc9c: reopening flushed file at 1731187542000 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a74c5f7: reopening flushed file at 1731187542005 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18994762: reopening flushed file at 1731187542011 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@445376a: reopening flushed file at 1731187542016 (+5 ms)Finished flush of dataSize ~51.44 KB/52675, heapSize ~63.33 KB/64848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 121ms, sequenceid=125, compaction requested=false at 1731187542023 (+7 ms)Writing region close event to WAL at 1731187542024 (+1 ms)Closed at 1731187542024
2024-11-09T21:25:42,024 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:42,024 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:42,024 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:42,025 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:42,025 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:42,026 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36959 is added to blk_1073741830_1006 (size=61344)
2024-11-09T21:25:42,026 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38675 is added to blk_1073741830_1006 (size=61344)
2024-11-09T21:25:42,027 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-09T21:25:42,027 INFO  [M:0;d86ff69d6719:46193 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-09T21:25:42,027 INFO  [M:0;d86ff69d6719:46193 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46193
2024-11-09T21:25:42,027 INFO  [M:0;d86ff69d6719:46193 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-09T21:25:42,161 INFO  [M:0;d86ff69d6719:46193 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-09T21:25:42,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:25:42,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46193-0x10122c683f90000, quorum=127.0.0.1:57808, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:25:42,166 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@56931997{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:25:42,167 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@19c2a7a5{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:25:42,167 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:25:42,167 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3c5fec98{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:25:42,167 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48123521{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/hadoop.log.dir/,STOPPED}
2024-11-09T21:25:42,169 WARN  [BP-1597026677-172.17.0.2-1731187501718 heartbeating to localhost/127.0.0.1:38249 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:25:42,169 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:25:42,169 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:25:42,169 WARN  [BP-1597026677-172.17.0.2-1731187501718 heartbeating to localhost/127.0.0.1:38249 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1597026677-172.17.0.2-1731187501718 (Datanode Uuid 7ad3e5e1-0d11-403e-9383-4d51fdcbf210) service to localhost/127.0.0.1:38249
2024-11-09T21:25:42,170 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/cluster_e839f11b-8a2a-e3b6-d5fe-08fab413cc82/data/data3/current/BP-1597026677-172.17.0.2-1731187501718 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:25:42,170 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/cluster_e839f11b-8a2a-e3b6-d5fe-08fab413cc82/data/data4/current/BP-1597026677-172.17.0.2-1731187501718 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:25:42,171 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:25:42,173 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@47bcda8c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:25:42,173 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@29765213{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:25:42,173 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:25:42,173 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@19fe8881{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:25:42,173 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@49490ce4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/hadoop.log.dir/,STOPPED}
2024-11-09T21:25:42,175 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:25:42,175 WARN  [BP-1597026677-172.17.0.2-1731187501718 heartbeating to localhost/127.0.0.1:38249 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:25:42,175 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:25:42,175 WARN  [BP-1597026677-172.17.0.2-1731187501718 heartbeating to localhost/127.0.0.1:38249 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1597026677-172.17.0.2-1731187501718 (Datanode Uuid fe225878-ad5d-47ea-bc05-6f4eede53cc5) service to localhost/127.0.0.1:38249
2024-11-09T21:25:42,176 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/cluster_e839f11b-8a2a-e3b6-d5fe-08fab413cc82/data/data1/current/BP-1597026677-172.17.0.2-1731187501718 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:25:42,176 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/cluster_e839f11b-8a2a-e3b6-d5fe-08fab413cc82/data/data2/current/BP-1597026677-172.17.0.2-1731187501718 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:25:42,176 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:25:42,182 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@59fd7f75{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-09T21:25:42,182 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4b9d1fb8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:25:42,182 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:25:42,182 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@64d2170c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:25:42,182 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b28e221{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/hadoop.log.dir/,STOPPED}
2024-11-09T21:25:42,189 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-09T21:25:42,217 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-09T21:25:42,226 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=233 (was 209)
Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38249 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38249
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38249 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38249
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38249
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38249 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:38249
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:38249
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=515 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=17 (was 22), ProcessCount=11 (was 11), AvailableMemoryMB=17938 (was 17992)
2024-11-09T21:25:42,233 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=233, OpenFileDescriptor=515, MaxFileDescriptor=1048576, SystemLoadAverage=17, ProcessCount=11, AvailableMemoryMB=17938
2024-11-09T21:25:42,233 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-09T21:25:42,233 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/hadoop.log.dir so I do NOT create it in target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42
2024-11-09T21:25:42,233 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a1ea1d95-8739-737b-90b0-c97d51a2e98f/hadoop.tmp.dir so I do NOT create it in target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42
2024-11-09T21:25:42,233 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/cluster_b81e8969-336e-2cb1-7e24-5e112673e34c, deleteOnExit=true
2024-11-09T21:25:42,233 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-09T21:25:42,233 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/test.cache.data in system properties and HBase conf
2024-11-09T21:25:42,233 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/hadoop.tmp.dir in system properties and HBase conf
2024-11-09T21:25:42,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/hadoop.log.dir in system properties and HBase conf
2024-11-09T21:25:42,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-09T21:25:42,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-09T21:25:42,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-09T21:25:42,234 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-09T21:25:42,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-09T21:25:42,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-09T21:25:42,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-09T21:25:42,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-09T21:25:42,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-09T21:25:42,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-09T21:25:42,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-09T21:25:42,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-09T21:25:42,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-09T21:25:42,234 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/nfs.dump.dir in system properties and HBase conf
2024-11-09T21:25:42,235 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/java.io.tmpdir in system properties and HBase conf
2024-11-09T21:25:42,235 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-09T21:25:42,235 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-09T21:25:42,235 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-09T21:25:42,248 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-09T21:25:42,329 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:42,530 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:25:42,533 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:25:42,534 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:25:42,534 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:25:42,534 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-09T21:25:42,534 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:25:42,535 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@12ac7c6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:25:42,535 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17951be7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:25:42,591 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:42,592 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:42,625 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1773ea07{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/java.io.tmpdir/jetty-localhost-41675-hadoop-hdfs-3_4_1-tests_jar-_-any-5131775606529561218/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-09T21:25:42,625 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@45235d1a{HTTP/1.1, (http/1.1)}{localhost:41675}
2024-11-09T21:25:42,625 INFO  [Time-limited test {}] server.Server(415): Started @290044ms
2024-11-09T21:25:42,636 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-09T21:25:42,829 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:25:42,832 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:25:42,832 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:25:42,832 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:25:42,832 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-09T21:25:42,833 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6ef29692{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:25:42,833 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@61ab51b1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:25:42,929 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@32b4a64d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/java.io.tmpdir/jetty-localhost-33091-hadoop-hdfs-3_4_1-tests_jar-_-any-3098715324413463697/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:25:42,929 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2ea5aa4c{HTTP/1.1, (http/1.1)}{localhost:33091}
2024-11-09T21:25:42,929 INFO  [Time-limited test {}] server.Server(415): Started @290348ms
2024-11-09T21:25:42,930 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:25:42,953 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-09T21:25:42,955 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-09T21:25:42,956 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-09T21:25:42,956 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-09T21:25:42,956 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-09T21:25:42,956 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a513798{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/hadoop.log.dir/,AVAILABLE}
2024-11-09T21:25:42,956 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@59dd2dec{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-09T21:25:43,052 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@9255255{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/java.io.tmpdir/jetty-localhost-36699-hadoop-hdfs-3_4_1-tests_jar-_-any-3222768579082859283/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:25:43,052 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2869fc81{HTTP/1.1, (http/1.1)}{localhost:36699}
2024-11-09T21:25:43,052 INFO  [Time-limited test {}] server.Server(415): Started @290471ms
2024-11-09T21:25:43,054 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-09T21:25:43,330 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:43,592 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:43,593 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:43,618 WARN  [Thread-2501 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/cluster_b81e8969-336e-2cb1-7e24-5e112673e34c/data/data1/current/BP-1286790564-172.17.0.2-1731187542251/current, will proceed with Du for space computation calculation, 
2024-11-09T21:25:43,618 WARN  [Thread-2502 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/cluster_b81e8969-336e-2cb1-7e24-5e112673e34c/data/data2/current/BP-1286790564-172.17.0.2-1731187542251/current, will proceed with Du for space computation calculation, 
2024-11-09T21:25:43,639 WARN  [Thread-2465 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:25:43,641 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa5492d4428689524 with lease ID 0xa3c851bef49183bf: Processing first storage report for DS-20e0af76-6e19-429b-bbcd-0bffafb903e3 from datanode DatanodeRegistration(127.0.0.1:37497, datanodeUuid=913b8f79-91b0-43f9-bc81-6d2868e02ff6, infoPort=45809, infoSecurePort=0, ipcPort=43591, storageInfo=lv=-57;cid=testClusterID;nsid=276642724;c=1731187542251)
2024-11-09T21:25:43,641 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa5492d4428689524 with lease ID 0xa3c851bef49183bf: from storage DS-20e0af76-6e19-429b-bbcd-0bffafb903e3 node DatanodeRegistration(127.0.0.1:37497, datanodeUuid=913b8f79-91b0-43f9-bc81-6d2868e02ff6, infoPort=45809, infoSecurePort=0, ipcPort=43591, storageInfo=lv=-57;cid=testClusterID;nsid=276642724;c=1731187542251), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:25:43,641 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa5492d4428689524 with lease ID 0xa3c851bef49183bf: Processing first storage report for DS-6f6515d1-5d9e-47f3-ad71-99727fb2458e from datanode DatanodeRegistration(127.0.0.1:37497, datanodeUuid=913b8f79-91b0-43f9-bc81-6d2868e02ff6, infoPort=45809, infoSecurePort=0, ipcPort=43591, storageInfo=lv=-57;cid=testClusterID;nsid=276642724;c=1731187542251)
2024-11-09T21:25:43,641 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa5492d4428689524 with lease ID 0xa3c851bef49183bf: from storage DS-6f6515d1-5d9e-47f3-ad71-99727fb2458e node DatanodeRegistration(127.0.0.1:37497, datanodeUuid=913b8f79-91b0-43f9-bc81-6d2868e02ff6, infoPort=45809, infoSecurePort=0, ipcPort=43591, storageInfo=lv=-57;cid=testClusterID;nsid=276642724;c=1731187542251), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:25:43,770 WARN  [Thread-2512 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/cluster_b81e8969-336e-2cb1-7e24-5e112673e34c/data/data3/current/BP-1286790564-172.17.0.2-1731187542251/current, will proceed with Du for space computation calculation, 
2024-11-09T21:25:43,770 WARN  [Thread-2513 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/cluster_b81e8969-336e-2cb1-7e24-5e112673e34c/data/data4/current/BP-1286790564-172.17.0.2-1731187542251/current, will proceed with Du for space computation calculation, 
2024-11-09T21:25:43,791 WARN  [Thread-2488 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-09T21:25:43,793 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9f8b85383454ede1 with lease ID 0xa3c851bef49183c0: Processing first storage report for DS-7931a879-a0c3-4f3c-b912-bd7e26aa9b9c from datanode DatanodeRegistration(127.0.0.1:37083, datanodeUuid=04a9b58a-b44f-4e1e-b725-7e07d7e5997c, infoPort=42729, infoSecurePort=0, ipcPort=32781, storageInfo=lv=-57;cid=testClusterID;nsid=276642724;c=1731187542251)
2024-11-09T21:25:43,793 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9f8b85383454ede1 with lease ID 0xa3c851bef49183c0: from storage DS-7931a879-a0c3-4f3c-b912-bd7e26aa9b9c node DatanodeRegistration(127.0.0.1:37083, datanodeUuid=04a9b58a-b44f-4e1e-b725-7e07d7e5997c, infoPort=42729, infoSecurePort=0, ipcPort=32781, storageInfo=lv=-57;cid=testClusterID;nsid=276642724;c=1731187542251), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:25:43,793 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9f8b85383454ede1 with lease ID 0xa3c851bef49183c0: Processing first storage report for DS-deeb3fc2-8950-4336-af50-8ea2c7651ab6 from datanode DatanodeRegistration(127.0.0.1:37083, datanodeUuid=04a9b58a-b44f-4e1e-b725-7e07d7e5997c, infoPort=42729, infoSecurePort=0, ipcPort=32781, storageInfo=lv=-57;cid=testClusterID;nsid=276642724;c=1731187542251)
2024-11-09T21:25:43,793 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9f8b85383454ede1 with lease ID 0xa3c851bef49183c0: from storage DS-deeb3fc2-8950-4336-af50-8ea2c7651ab6 node DatanodeRegistration(127.0.0.1:37083, datanodeUuid=04a9b58a-b44f-4e1e-b725-7e07d7e5997c, infoPort=42729, infoSecurePort=0, ipcPort=32781, storageInfo=lv=-57;cid=testClusterID;nsid=276642724;c=1731187542251), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-09T21:25:43,880 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42
2024-11-09T21:25:43,885 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/cluster_b81e8969-336e-2cb1-7e24-5e112673e34c/zookeeper_0, clientPort=61635, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/cluster_b81e8969-336e-2cb1-7e24-5e112673e34c/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/cluster_b81e8969-336e-2cb1-7e24-5e112673e34c/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-09T21:25:43,887 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61635
2024-11-09T21:25:43,887 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:25:43,889 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:25:43,899 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741825_1001 (size=7)
2024-11-09T21:25:43,899 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741825_1001 (size=7)
2024-11-09T21:25:43,900 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9 with version=8
2024-11-09T21:25:43,900 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34451/user/jenkins/test-data/a8b002c3-b570-7a41-9e97-fcac78af79e8/hbase-staging
2024-11-09T21:25:43,902 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d86ff69d6719:0 server-side Connection retries=45
2024-11-09T21:25:43,902 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:25:43,902 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T21:25:43,902 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T21:25:43,902 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:25:43,902 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T21:25:43,902 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-09T21:25:43,903 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T21:25:43,903 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36821
2024-11-09T21:25:43,904 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36821 connecting to ZooKeeper ensemble=127.0.0.1:61635
2024-11-09T21:25:43,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:368210x0, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T21:25:43,976 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36821-0x10122c722110000 connected
2024-11-09T21:25:44,035 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:25:44,036 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:25:44,038 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:25:44,038 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9, hbase.cluster.distributed=false
2024-11-09T21:25:44,041 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T21:25:44,041 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36821
2024-11-09T21:25:44,041 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36821
2024-11-09T21:25:44,042 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36821
2024-11-09T21:25:44,042 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36821
2024-11-09T21:25:44,043 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36821
2024-11-09T21:25:44,059 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d86ff69d6719:0 server-side Connection retries=45
2024-11-09T21:25:44,060 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:25:44,060 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-09T21:25:44,060 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-09T21:25:44,060 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-09T21:25:44,060 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-09T21:25:44,060 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-09T21:25:44,060 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-09T21:25:44,060 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46375
2024-11-09T21:25:44,061 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46375 connecting to ZooKeeper ensemble=127.0.0.1:61635
2024-11-09T21:25:44,062 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:25:44,063 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:25:44,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:463750x0, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-09T21:25:44,076 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46375-0x10122c722110001 connected
2024-11-09T21:25:44,076 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:25:44,076 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-09T21:25:44,077 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-09T21:25:44,078 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-09T21:25:44,079 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-09T21:25:44,080 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46375
2024-11-09T21:25:44,080 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46375
2024-11-09T21:25:44,080 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46375
2024-11-09T21:25:44,080 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46375
2024-11-09T21:25:44,081 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46375
2024-11-09T21:25:44,096 DEBUG [M:0;d86ff69d6719:36821 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d86ff69d6719:36821
2024-11-09T21:25:44,097 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d86ff69d6719,36821,1731187543902
2024-11-09T21:25:44,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:25:44,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:25:44,102 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d86ff69d6719,36821,1731187543902
2024-11-09T21:25:44,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-09T21:25:44,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:44,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:44,110 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-09T21:25:44,111 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d86ff69d6719,36821,1731187543902 from backup master directory
2024-11-09T21:25:44,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d86ff69d6719,36821,1731187543902
2024-11-09T21:25:44,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:25:44,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-09T21:25:44,118 WARN  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T21:25:44,118 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d86ff69d6719,36821,1731187543902
2024-11-09T21:25:44,123 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/hbase.id] with ID: d6f36194-da35-4da8-97eb-702cf1037fab
2024-11-09T21:25:44,123 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/.tmp/hbase.id
2024-11-09T21:25:44,129 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741826_1002 (size=42)
2024-11-09T21:25:44,130 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741826_1002 (size=42)
2024-11-09T21:25:44,130 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/.tmp/hbase.id]:[hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/hbase.id]
2024-11-09T21:25:44,140 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:25:44,140 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-09T21:25:44,141 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-09T21:25:44,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:44,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:44,156 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741827_1003 (size=196)
2024-11-09T21:25:44,157 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741827_1003 (size=196)
2024-11-09T21:25:44,157 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-09T21:25:44,158 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-09T21:25:44,158 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:25:44,163 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741828_1004 (size=1189)
2024-11-09T21:25:44,164 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741828_1004 (size=1189)
2024-11-09T21:25:44,164 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store
2024-11-09T21:25:44,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741829_1005 (size=34)
2024-11-09T21:25:44,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741829_1005 (size=34)
2024-11-09T21:25:44,170 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:25:44,170 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-09T21:25:44,170 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:44,170 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:44,170 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-09T21:25:44,170 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:44,170 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:44,170 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731187544170Disabling compacts and flushes for region at 1731187544170Disabling writes for close at 1731187544170Writing region close event to WAL at 1731187544170Closed at 1731187544170
2024-11-09T21:25:44,171 WARN  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/.initializing
2024-11-09T21:25:44,171 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/WALs/d86ff69d6719,36821,1731187543902
2024-11-09T21:25:44,173 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C36821%2C1731187543902, suffix=, logDir=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/WALs/d86ff69d6719,36821,1731187543902, archiveDir=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/oldWALs, maxLogs=10
2024-11-09T21:25:44,173 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C36821%2C1731187543902.1731187544173
2024-11-09T21:25:44,177 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/WALs/d86ff69d6719,36821,1731187543902/d86ff69d6719%2C36821%2C1731187543902.1731187544173
2024-11-09T21:25:44,178 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45809:45809),(127.0.0.1/127.0.0.1:42729:42729)]
2024-11-09T21:25:44,181 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:25:44,181 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:25:44,181 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:25:44,181 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:25:44,182 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:25:44,184 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-09T21:25:44,184 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:44,184 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:44,184 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:25:44,185 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-09T21:25:44,185 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:44,185 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:25:44,186 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:25:44,186 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-09T21:25:44,186 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:44,187 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:25:44,187 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-09T21:25:44,188 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-09T21:25:44,188 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:44,188 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-09T21:25:44,189 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:25:44,189 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:25:44,189 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:25:44,190 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:25:44,190 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:25:44,191 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-09T21:25:44,192 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-09T21:25:44,194 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:25:44,195 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=749563, jitterRate=-0.04688137769699097}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-09T21:25:44,195 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731187544181Initializing all the Stores at 1731187544182 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187544182Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187544182Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187544182Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187544182Cleaning up temporary data from old regions at 1731187544190 (+8 ms)Region opened successfully at 1731187544195 (+5 ms)
2024-11-09T21:25:44,195 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-09T21:25:44,198 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@760d9be3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0
2024-11-09T21:25:44,198 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-09T21:25:44,198 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-09T21:25:44,199 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-09T21:25:44,199 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-09T21:25:44,199 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-09T21:25:44,199 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-09T21:25:44,199 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-09T21:25:44,203 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-09T21:25:44,204 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-09T21:25:44,242 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-09T21:25:44,243 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-09T21:25:44,244 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-09T21:25:44,252 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-09T21:25:44,252 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-09T21:25:44,253 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-09T21:25:44,260 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-09T21:25:44,261 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-09T21:25:44,268 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-09T21:25:44,272 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-09T21:25:44,277 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-09T21:25:44,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-09T21:25:44,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-09T21:25:44,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:44,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:44,285 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d86ff69d6719,36821,1731187543902, sessionid=0x10122c722110000, setting cluster-up flag (Was=false)
2024-11-09T21:25:44,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:44,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:44,327 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-09T21:25:44,329 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d86ff69d6719,36821,1731187543902
2024-11-09T21:25:44,331 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:44,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:44,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:44,369 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-09T21:25:44,372 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d86ff69d6719,36821,1731187543902
2024-11-09T21:25:44,375 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-09T21:25:44,378 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-09T21:25:44,378 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-09T21:25:44,379 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-09T21:25:44,379 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d86ff69d6719,36821,1731187543902
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-09T21:25:44,381 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:25:44,381 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:25:44,381 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:25:44,381 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5
2024-11-09T21:25:44,381 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d86ff69d6719:0, corePoolSize=10, maxPoolSize=10
2024-11-09T21:25:44,381 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:44,381 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2
2024-11-09T21:25:44,382 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:44,383 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731187574383
2024-11-09T21:25:44,383 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.HRegionServer(746): ClusterId : d6f36194-da35-4da8-97eb-702cf1037fab
2024-11-09T21:25:44,383 DEBUG [RS:0;d86ff69d6719:46375 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-09T21:25:44,383 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-09T21:25:44,383 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-09T21:25:44,383 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-09T21:25:44,383 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-09T21:25:44,383 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-09T21:25:44,384 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-09T21:25:44,384 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:44,384 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:25:44,384 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-09T21:25:44,385 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:44,386 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-09T21:25:44,386 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-09T21:25:44,386 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-09T21:25:44,386 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-09T21:25:44,387 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-09T21:25:44,387 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-09T21:25:44,387 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187544387,5,FailOnTimeoutGroup]
2024-11-09T21:25:44,387 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187544387,5,FailOnTimeoutGroup]
2024-11-09T21:25:44,387 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:44,387 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-09T21:25:44,387 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:44,387 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:44,393 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741831_1007 (size=1321)
2024-11-09T21:25:44,393 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741831_1007 (size=1321)
2024-11-09T21:25:44,394 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-09T21:25:44,394 DEBUG [RS:0;d86ff69d6719:46375 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-09T21:25:44,394 DEBUG [RS:0;d86ff69d6719:46375 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-09T21:25:44,394 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9
2024-11-09T21:25:44,399 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741832_1008 (size=32)
2024-11-09T21:25:44,399 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741832_1008 (size=32)
2024-11-09T21:25:44,400 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:25:44,401 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-09T21:25:44,402 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-09T21:25:44,402 DEBUG [RS:0;d86ff69d6719:46375 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-09T21:25:44,402 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:44,403 DEBUG [RS:0;d86ff69d6719:46375 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52bfa3c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0
2024-11-09T21:25:44,403 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:44,403 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-09T21:25:44,405 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-09T21:25:44,405 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:44,405 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:44,405 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-09T21:25:44,407 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-09T21:25:44,407 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:44,407 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:44,407 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-09T21:25:44,409 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-09T21:25:44,409 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:44,409 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:44,409 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-09T21:25:44,410 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/data/hbase/meta/1588230740
2024-11-09T21:25:44,410 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/data/hbase/meta/1588230740
2024-11-09T21:25:44,411 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-09T21:25:44,411 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-09T21:25:44,412 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-09T21:25:44,413 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-09T21:25:44,414 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-09T21:25:44,415 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=787226, jitterRate=0.0010107606649398804}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-09T21:25:44,415 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731187544400Initializing all the Stores at 1731187544401 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187544401Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187544401Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187544401Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187544401Cleaning up temporary data from old regions at 1731187544411 (+10 ms)Region opened successfully at 1731187544415 (+4 ms)
2024-11-09T21:25:44,415 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-09T21:25:44,416 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-09T21:25:44,416 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-09T21:25:44,416 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-09T21:25:44,416 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-09T21:25:44,416 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-09T21:25:44,416 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731187544415Disabling compacts and flushes for region at 1731187544415Disabling writes for close at 1731187544416 (+1 ms)Writing region close event to WAL at 1731187544416Closed at 1731187544416
2024-11-09T21:25:44,417 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:25:44,417 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-09T21:25:44,417 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-09T21:25:44,418 DEBUG [RS:0;d86ff69d6719:46375 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d86ff69d6719:46375
2024-11-09T21:25:44,418 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-09T21:25:44,418 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-09T21:25:44,418 DEBUG [RS:0;d86ff69d6719:46375 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-09T21:25:44,418 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-09T21:25:44,418 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.HRegionServer(2659): reportForDuty to master=d86ff69d6719,36821,1731187543902 with port=46375, startcode=1731187544059
2024-11-09T21:25:44,418 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-09T21:25:44,419 DEBUG [RS:0;d86ff69d6719:46375 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-09T21:25:44,423 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41609, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-11-09T21:25:44,423 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36821 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d86ff69d6719,46375,1731187544059
2024-11-09T21:25:44,423 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36821 {}] master.ServerManager(517): Registering regionserver=d86ff69d6719,46375,1731187544059
2024-11-09T21:25:44,425 DEBUG [RS:0;d86ff69d6719:46375 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9
2024-11-09T21:25:44,425 DEBUG [RS:0;d86ff69d6719:46375 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45487
2024-11-09T21:25:44,425 DEBUG [RS:0;d86ff69d6719:46375 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-09T21:25:44,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T21:25:44,436 DEBUG [RS:0;d86ff69d6719:46375 {}] zookeeper.ZKUtil(111): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d86ff69d6719,46375,1731187544059
2024-11-09T21:25:44,436 WARN  [RS:0;d86ff69d6719:46375 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-09T21:25:44,436 INFO  [RS:0;d86ff69d6719:46375 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:25:44,436 DEBUG [RS:0;d86ff69d6719:46375 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/WALs/d86ff69d6719,46375,1731187544059
2024-11-09T21:25:44,436 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d86ff69d6719,46375,1731187544059]
2024-11-09T21:25:44,440 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-09T21:25:44,442 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-09T21:25:44,442 INFO  [RS:0;d86ff69d6719:46375 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-09T21:25:44,442 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:44,442 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-09T21:25:44,443 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-09T21:25:44,443 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:44,444 DEBUG [RS:0;d86ff69d6719:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:44,444 DEBUG [RS:0;d86ff69d6719:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:44,444 DEBUG [RS:0;d86ff69d6719:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:44,444 DEBUG [RS:0;d86ff69d6719:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:44,444 DEBUG [RS:0;d86ff69d6719:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:44,444 DEBUG [RS:0;d86ff69d6719:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2
2024-11-09T21:25:44,444 DEBUG [RS:0;d86ff69d6719:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:44,444 DEBUG [RS:0;d86ff69d6719:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:44,444 DEBUG [RS:0;d86ff69d6719:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:44,444 DEBUG [RS:0;d86ff69d6719:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:44,444 DEBUG [RS:0;d86ff69d6719:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:44,444 DEBUG [RS:0;d86ff69d6719:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1
2024-11-09T21:25:44,444 DEBUG [RS:0;d86ff69d6719:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3
2024-11-09T21:25:44,444 DEBUG [RS:0;d86ff69d6719:46375 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3
2024-11-09T21:25:44,445 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:44,445 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:44,445 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:44,445 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:44,445 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:44,445 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46375,1731187544059-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T21:25:44,462 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-09T21:25:44,462 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46375,1731187544059-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:44,462 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:44,462 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.Replication(171): d86ff69d6719,46375,1731187544059 started
2024-11-09T21:25:44,473 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:44,473 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.HRegionServer(1482): Serving as d86ff69d6719,46375,1731187544059, RpcServer on d86ff69d6719/172.17.0.2:46375, sessionid=0x10122c722110001
2024-11-09T21:25:44,473 DEBUG [RS:0;d86ff69d6719:46375 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-09T21:25:44,473 DEBUG [RS:0;d86ff69d6719:46375 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d86ff69d6719,46375,1731187544059
2024-11-09T21:25:44,473 DEBUG [RS:0;d86ff69d6719:46375 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,46375,1731187544059'
2024-11-09T21:25:44,473 DEBUG [RS:0;d86ff69d6719:46375 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-09T21:25:44,474 DEBUG [RS:0;d86ff69d6719:46375 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-09T21:25:44,474 DEBUG [RS:0;d86ff69d6719:46375 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-09T21:25:44,474 DEBUG [RS:0;d86ff69d6719:46375 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-09T21:25:44,474 DEBUG [RS:0;d86ff69d6719:46375 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d86ff69d6719,46375,1731187544059
2024-11-09T21:25:44,474 DEBUG [RS:0;d86ff69d6719:46375 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,46375,1731187544059'
2024-11-09T21:25:44,474 DEBUG [RS:0;d86ff69d6719:46375 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-09T21:25:44,475 DEBUG [RS:0;d86ff69d6719:46375 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-09T21:25:44,475 DEBUG [RS:0;d86ff69d6719:46375 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-09T21:25:44,475 INFO  [RS:0;d86ff69d6719:46375 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-09T21:25:44,475 INFO  [RS:0;d86ff69d6719:46375 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-09T21:25:44,569 WARN  [d86ff69d6719:36821 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-09T21:25:44,578 INFO  [RS:0;d86ff69d6719:46375 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C46375%2C1731187544059, suffix=, logDir=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/WALs/d86ff69d6719,46375,1731187544059, archiveDir=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/oldWALs, maxLogs=32
2024-11-09T21:25:44,579 INFO  [RS:0;d86ff69d6719:46375 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C46375%2C1731187544059.1731187544579
2024-11-09T21:25:44,588 INFO  [RS:0;d86ff69d6719:46375 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/WALs/d86ff69d6719,46375,1731187544059/d86ff69d6719%2C46375%2C1731187544059.1731187544579
2024-11-09T21:25:44,589 DEBUG [RS:0;d86ff69d6719:46375 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42729:42729),(127.0.0.1/127.0.0.1:45809:45809)]
2024-11-09T21:25:44,592 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:44,593 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:44,819 DEBUG [d86ff69d6719:36821 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-09T21:25:44,820 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d86ff69d6719,46375,1731187544059
2024-11-09T21:25:44,824 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d86ff69d6719,46375,1731187544059, state=OPENING
2024-11-09T21:25:44,859 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-09T21:25:44,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:44,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:44,871 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-09T21:25:44,871 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:25:44,871 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:25:44,871 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d86ff69d6719,46375,1731187544059}]
2024-11-09T21:25:45,028 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-09T21:25:45,033 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46843, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-09T21:25:45,038 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-09T21:25:45,039 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:25:45,042 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C46375%2C1731187544059.meta, suffix=.meta, logDir=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/WALs/d86ff69d6719,46375,1731187544059, archiveDir=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/oldWALs, maxLogs=32
2024-11-09T21:25:45,042 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d86ff69d6719%2C46375%2C1731187544059.meta.1731187545042.meta
2024-11-09T21:25:45,048 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/WALs/d86ff69d6719,46375,1731187544059/d86ff69d6719%2C46375%2C1731187544059.meta.1731187545042.meta
2024-11-09T21:25:45,054 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45809:45809),(127.0.0.1/127.0.0.1:42729:42729)]
2024-11-09T21:25:45,055 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-09T21:25:45,056 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-09T21:25:45,056 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-09T21:25:45,056 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-09T21:25:45,056 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-09T21:25:45,056 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-09T21:25:45,056 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-09T21:25:45,056 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-09T21:25:45,058 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-09T21:25:45,059 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-09T21:25:45,059 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:45,060 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:45,060 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-09T21:25:45,060 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-09T21:25:45,060 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:45,061 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:45,061 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-09T21:25:45,061 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-09T21:25:45,061 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:45,061 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:45,061 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-09T21:25:45,062 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-09T21:25:45,062 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-09T21:25:45,062 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-09T21:25:45,062 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-09T21:25:45,063 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/data/hbase/meta/1588230740
2024-11-09T21:25:45,064 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/data/hbase/meta/1588230740
2024-11-09T21:25:45,065 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-09T21:25:45,065 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-09T21:25:45,065 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-09T21:25:45,066 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-09T21:25:45,067 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=730039, jitterRate=-0.07170864939689636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-09T21:25:45,067 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-09T21:25:45,067 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731187545056Writing region info on filesystem at 1731187545056Initializing all the Stores at 1731187545057 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187545057Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187545058 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187545058Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731187545058Cleaning up temporary data from old regions at 1731187545065 (+7 ms)Running coprocessor post-open hooks at 1731187545067 (+2 ms)Region opened successfully at 1731187545067
2024-11-09T21:25:45,068 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731187545027
2024-11-09T21:25:45,070 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-09T21:25:45,070 INFO  [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-09T21:25:45,071 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,46375,1731187544059
2024-11-09T21:25:45,071 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d86ff69d6719,46375,1731187544059, state=OPEN
2024-11-09T21:25:45,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-09T21:25:45,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-09T21:25:45,109 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d86ff69d6719,46375,1731187544059
2024-11-09T21:25:45,109 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:25:45,109 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-09T21:25:45,112 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-09T21:25:45,112 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d86ff69d6719,46375,1731187544059 in 238 msec
2024-11-09T21:25:45,116 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-09T21:25:45,116 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 695 msec
2024-11-09T21:25:45,118 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-09T21:25:45,118 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-09T21:25:45,120 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-09T21:25:45,120 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,46375,1731187544059, seqNum=-1]
2024-11-09T21:25:45,120 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T21:25:45,122 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59309, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T21:25:45,130 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 751 msec
2024-11-09T21:25:45,130 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731187545130, completionTime=-1
2024-11-09T21:25:45,130 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-09T21:25:45,130 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-09T21:25:45,132 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-09T21:25:45,132 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731187605132
2024-11-09T21:25:45,133 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731187665133
2024-11-09T21:25:45,133 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-09T21:25:45,133 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,36821,1731187543902-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:45,133 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,36821,1731187543902-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:45,133 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,36821,1731187543902-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:45,134 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d86ff69d6719:36821, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:45,134 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:45,134 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:45,136 DEBUG [master/d86ff69d6719:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-09T21:25:45,139 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.021sec
2024-11-09T21:25:45,139 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-09T21:25:45,139 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-09T21:25:45,139 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-09T21:25:45,139 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-09T21:25:45,139 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-09T21:25:45,139 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,36821,1731187543902-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-09T21:25:45,139 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,36821,1731187543902-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-09T21:25:45,142 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-09T21:25:45,142 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-09T21:25:45,142 INFO  [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,36821,1731187543902-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-09T21:25:45,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f36ce3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:25:45,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d86ff69d6719,36821,-1 for getting cluster id
2024-11-09T21:25:45,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-09T21:25:45,185 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd6f36194-da35-4da8-97eb-702cf1037fab'
2024-11-09T21:25:45,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-09T21:25:45,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d6f36194-da35-4da8-97eb-702cf1037fab"

2024-11-09T21:25:45,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba6c542, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:25:45,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d86ff69d6719,36821,-1]
2024-11-09T21:25:45,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-09T21:25:45,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:45,188 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-09T21:25:45,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@338da4b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-09T21:25:45,190 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-09T21:25:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,46375,1731187544059, seqNum=-1]
2024-11-09T21:25:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-09T21:25:45,193 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-09T21:25:45,196 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d86ff69d6719,36821,1731187543902
2024-11-09T21:25:45,197 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-09T21:25:45,200 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-09T21:25:45,200 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-09T21:25:45,203 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/WALs/test.com,8080,1, archiveDir=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/oldWALs, maxLogs=32
2024-11-09T21:25:45,204 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731187545204
2024-11-09T21:25:45,211 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/WALs/test.com,8080,1/test.com%2C8080%2C1.1731187545204
2024-11-09T21:25:45,212 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42729:42729),(127.0.0.1/127.0.0.1:45809:45809)]
2024-11-09T21:25:45,213 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731187545213
2024-11-09T21:25:45,219 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,220 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,220 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,220 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,220 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,220 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/WALs/test.com,8080,1/test.com%2C8080%2C1.1731187545204 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/WALs/test.com,8080,1/test.com%2C8080%2C1.1731187545213
2024-11-09T21:25:45,221 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45809:45809),(127.0.0.1/127.0.0.1:42729:42729)]
2024-11-09T21:25:45,222 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/WALs/test.com,8080,1/test.com%2C8080%2C1.1731187545204 is not closed yet, will try archiving it next time
2024-11-09T21:25:45,222 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,222 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,222 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741835_1011 (size=93)
2024-11-09T21:25:45,223 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,223 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741835_1011 (size=93)
2024-11-09T21:25:45,223 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,223 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,225 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741836_1012 (size=93)
2024-11-09T21:25:45,226 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/WALs/test.com,8080,1/test.com%2C8080%2C1.1731187545204 to hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/oldWALs/test.com%2C8080%2C1.1731187545204
2024-11-09T21:25:45,226 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741836_1012 (size=93)
2024-11-09T21:25:45,229 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/oldWALs
2024-11-09T21:25:45,229 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1731187545213)
2024-11-09T21:25:45,229 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-09T21:25:45,229 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-09T21:25:45,229 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:25:45,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:45,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:45,229 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-09T21:25:45,230 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-09T21:25:45,230 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1744082799, stopped=false
2024-11-09T21:25:45,230 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d86ff69d6719,36821,1731187543902
2024-11-09T21:25:45,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-09T21:25:45,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-09T21:25:45,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:45,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:45,251 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-09T21:25:45,251 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-09T21:25:45,251 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:25:45,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:45,251 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:25:45,251 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-09T21:25:45,251 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd86ff69d6719,46375,1731187544059' *****
2024-11-09T21:25:45,251 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-09T21:25:45,252 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-09T21:25:45,252 INFO  [RS:0;d86ff69d6719:46375 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-09T21:25:45,252 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-09T21:25:45,252 INFO  [RS:0;d86ff69d6719:46375 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-09T21:25:45,252 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.HRegionServer(959): stopping server d86ff69d6719,46375,1731187544059
2024-11-09T21:25:45,252 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-09T21:25:45,252 INFO  [RS:0;d86ff69d6719:46375 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d86ff69d6719:46375.
2024-11-09T21:25:45,252 DEBUG [RS:0;d86ff69d6719:46375 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-09T21:25:45,252 DEBUG [RS:0;d86ff69d6719:46375 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:45,252 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-09T21:25:45,252 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-09T21:25:45,252 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-09T21:25:45,252 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-09T21:25:45,253 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-09T21:25:45,253 DEBUG [RS:0;d86ff69d6719:46375 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-09T21:25:45,253 DEBUG [RS:0;d86ff69d6719:46375 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-09T21:25:45,253 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-09T21:25:45,253 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-09T21:25:45,253 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-09T21:25:45,253 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-09T21:25:45,253 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-09T21:25:45,253 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-09T21:25:45,271 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/data/hbase/meta/1588230740/.tmp/ns/5cc93617c6c2468f8e99f7f0b4c86ae1 is 43, key is default/ns:d/1731187545123/Put/seqid=0
2024-11-09T21:25:45,275 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741837_1013 (size=5153)
2024-11-09T21:25:45,275 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741837_1013 (size=5153)
2024-11-09T21:25:45,276 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/data/hbase/meta/1588230740/.tmp/ns/5cc93617c6c2468f8e99f7f0b4c86ae1
2024-11-09T21:25:45,283 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/data/hbase/meta/1588230740/.tmp/ns/5cc93617c6c2468f8e99f7f0b4c86ae1 as hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/data/hbase/meta/1588230740/ns/5cc93617c6c2468f8e99f7f0b4c86ae1
2024-11-09T21:25:45,287 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/data/hbase/meta/1588230740/ns/5cc93617c6c2468f8e99f7f0b4c86ae1, entries=2, sequenceid=6, filesize=5.0 K
2024-11-09T21:25:45,288 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 35ms, sequenceid=6, compaction requested=false
2024-11-09T21:25:45,292 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-09T21:25:45,293 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-09T21:25:45,293 INFO  [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-09T21:25:45,293 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731187545253Running coprocessor pre-close hooks at 1731187545253Disabling compacts and flushes for region at 1731187545253Disabling writes for close at 1731187545253Obtaining lock to block concurrent updates at 1731187545253Preparing flush snapshotting stores in 1588230740 at 1731187545253Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731187545253Flushing stores of hbase:meta,,1.1588230740 at 1731187545254 (+1 ms)Flushing 1588230740/ns: creating writer at 1731187545254Flushing 1588230740/ns: appending metadata at 1731187545271 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731187545271Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d59ead8: reopening flushed file at 1731187545282 (+11 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 35ms, sequenceid=6, compaction requested=false at 1731187545288 (+6 ms)Writing region close event to WAL at 1731187545289 (+1 ms)Running coprocessor post-close hooks at 1731187545293 (+4 ms)Closed at 1731187545293
2024-11-09T21:25:45,293 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-09T21:25:45,331 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.1731187361880
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:45,446 INFO  [regionserver/d86ff69d6719:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-09T21:25:45,446 INFO  [regionserver/d86ff69d6719:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-09T21:25:45,453 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.HRegionServer(976): stopping server d86ff69d6719,46375,1731187544059; all regions closed.
2024-11-09T21:25:45,454 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,454 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,455 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,455 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,455 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,460 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741834_1010 (size=1152)
2024-11-09T21:25:45,460 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741834_1010 (size=1152)
2024-11-09T21:25:45,465 DEBUG [RS:0;d86ff69d6719:46375 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/oldWALs
2024-11-09T21:25:45,465 INFO  [RS:0;d86ff69d6719:46375 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d86ff69d6719%2C46375%2C1731187544059.meta:.meta(num 1731187545042)
2024-11-09T21:25:45,466 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,466 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,466 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,466 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,466 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741833_1009 (size=93)
2024-11-09T21:25:45,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741833_1009 (size=93)
2024-11-09T21:25:45,472 DEBUG [RS:0;d86ff69d6719:46375 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/oldWALs
2024-11-09T21:25:45,472 INFO  [RS:0;d86ff69d6719:46375 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d86ff69d6719%2C46375%2C1731187544059:(num 1731187544579)
2024-11-09T21:25:45,472 DEBUG [RS:0;d86ff69d6719:46375 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-09T21:25:45,472 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.LeaseManager(133): Closed leases
2024-11-09T21:25:45,473 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-09T21:25:45,473 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.ChoreService(370): Chore service for: regionserver/d86ff69d6719:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-09T21:25:45,473 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-09T21:25:45,473 INFO  [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-09T21:25:45,473 INFO  [RS:0;d86ff69d6719:46375 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46375
2024-11-09T21:25:45,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-09T21:25:45,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d86ff69d6719,46375,1731187544059
2024-11-09T21:25:45,494 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-09T21:25:45,495 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d86ff69d6719,46375,1731187544059]
2024-11-09T21:25:45,510 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d86ff69d6719,46375,1731187544059 already deleted, retry=false
2024-11-09T21:25:45,510 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d86ff69d6719,46375,1731187544059 expired; onlineServers=0
2024-11-09T21:25:45,510 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd86ff69d6719,36821,1731187543902' *****
2024-11-09T21:25:45,510 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-09T21:25:45,510 INFO  [M:0;d86ff69d6719:36821 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-09T21:25:45,510 INFO  [M:0;d86ff69d6719:36821 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-09T21:25:45,510 DEBUG [M:0;d86ff69d6719:36821 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-09T21:25:45,510 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-09T21:25:45,510 DEBUG [M:0;d86ff69d6719:36821 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-09T21:25:45,510 DEBUG [master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187544387 {}] cleaner.HFileCleaner(306): Exit Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731187544387,5,FailOnTimeoutGroup]
2024-11-09T21:25:45,510 DEBUG [master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187544387 {}] cleaner.HFileCleaner(306): Exit Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731187544387,5,FailOnTimeoutGroup]
2024-11-09T21:25:45,510 INFO  [M:0;d86ff69d6719:36821 {}] hbase.ChoreService(370): Chore service for: master/d86ff69d6719:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-09T21:25:45,510 INFO  [M:0;d86ff69d6719:36821 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-09T21:25:45,510 DEBUG [M:0;d86ff69d6719:36821 {}] master.HMaster(1795): Stopping service threads
2024-11-09T21:25:45,510 INFO  [M:0;d86ff69d6719:36821 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-09T21:25:45,510 INFO  [M:0;d86ff69d6719:36821 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-09T21:25:45,510 INFO  [M:0;d86ff69d6719:36821 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-09T21:25:45,511 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-09T21:25:45,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-09T21:25:45,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-09T21:25:45,518 DEBUG [M:0;d86ff69d6719:36821 {}] zookeeper.ZKUtil(347): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-09T21:25:45,518 WARN  [M:0;d86ff69d6719:36821 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-09T21:25:45,519 INFO  [M:0;d86ff69d6719:36821 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/.lastflushedseqids
2024-11-09T21:25:45,523 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741838_1014 (size=99)
2024-11-09T21:25:45,523 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741838_1014 (size=99)
2024-11-09T21:25:45,523 INFO  [M:0;d86ff69d6719:36821 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-09T21:25:45,524 INFO  [M:0;d86ff69d6719:36821 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-09T21:25:45,524 DEBUG [M:0;d86ff69d6719:36821 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-09T21:25:45,524 INFO  [M:0;d86ff69d6719:36821 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:45,524 DEBUG [M:0;d86ff69d6719:36821 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:45,524 DEBUG [M:0;d86ff69d6719:36821 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-09T21:25:45,524 DEBUG [M:0;d86ff69d6719:36821 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:45,524 INFO  [M:0;d86ff69d6719:36821 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-09T21:25:45,541 DEBUG [M:0;d86ff69d6719:36821 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ad73dec1254c4be1a1d2a673ff871f43 is 82, key is hbase:meta,,1/info:regioninfo/1731187545070/Put/seqid=0
2024-11-09T21:25:45,545 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741839_1015 (size=5672)
2024-11-09T21:25:45,545 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741839_1015 (size=5672)
2024-11-09T21:25:45,546 INFO  [M:0;d86ff69d6719:36821 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ad73dec1254c4be1a1d2a673ff871f43
2024-11-09T21:25:45,563 DEBUG [M:0;d86ff69d6719:36821 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/19a07b557bfa435180161896ed5830ac is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731187545129/Put/seqid=0
2024-11-09T21:25:45,566 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741840_1016 (size=5275)
2024-11-09T21:25:45,567 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741840_1016 (size=5275)
2024-11-09T21:25:45,567 INFO  [M:0;d86ff69d6719:36821 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/19a07b557bfa435180161896ed5830ac
2024-11-09T21:25:45,583 DEBUG [M:0;d86ff69d6719:36821 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b532410b94c34c58b086a30c939a3d0d is 69, key is d86ff69d6719,46375,1731187544059/rs:state/1731187544424/Put/seqid=0
2024-11-09T21:25:45,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741841_1017 (size=5156)
2024-11-09T21:25:45,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741841_1017 (size=5156)
2024-11-09T21:25:45,587 INFO  [M:0;d86ff69d6719:36821 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b532410b94c34c58b086a30c939a3d0d
2024-11-09T21:25:45,593 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,40715,1731187362517/d86ff69d6719%2C40715%2C1731187362517.1731187362735
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:45,593 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42197/user/jenkins/test-data/992fadee-ba25-24e9-1f71-e1cfc690351a/WALs/d86ff69d6719,37671,1731187361371/d86ff69d6719%2C37671%2C1731187361371.meta.1731187362371.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-09T21:25:45,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:25:45,602 INFO  [RS:0;d86ff69d6719:46375 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-09T21:25:45,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46375-0x10122c722110001, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:25:45,602 INFO  [RS:0;d86ff69d6719:46375 {}] regionserver.HRegionServer(1031): Exiting; stopping=d86ff69d6719,46375,1731187544059; zookeeper connection closed.
2024-11-09T21:25:45,602 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@71eef4fd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@71eef4fd
2024-11-09T21:25:45,603 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-09T21:25:45,603 DEBUG [M:0;d86ff69d6719:36821 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6fc96ce4387f4ae595755cea2ac2eed4 is 52, key is load_balancer_on/state:d/1731187545199/Put/seqid=0
2024-11-09T21:25:45,607 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741842_1018 (size=5056)
2024-11-09T21:25:45,607 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741842_1018 (size=5056)
2024-11-09T21:25:45,608 INFO  [M:0;d86ff69d6719:36821 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6fc96ce4387f4ae595755cea2ac2eed4
2024-11-09T21:25:45,612 DEBUG [M:0;d86ff69d6719:36821 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ad73dec1254c4be1a1d2a673ff871f43 as hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ad73dec1254c4be1a1d2a673ff871f43
2024-11-09T21:25:45,616 INFO  [M:0;d86ff69d6719:36821 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ad73dec1254c4be1a1d2a673ff871f43, entries=8, sequenceid=29, filesize=5.5 K
2024-11-09T21:25:45,617 DEBUG [M:0;d86ff69d6719:36821 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/19a07b557bfa435180161896ed5830ac as hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/19a07b557bfa435180161896ed5830ac
2024-11-09T21:25:45,620 INFO  [M:0;d86ff69d6719:36821 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/19a07b557bfa435180161896ed5830ac, entries=3, sequenceid=29, filesize=5.2 K
2024-11-09T21:25:45,621 DEBUG [M:0;d86ff69d6719:36821 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b532410b94c34c58b086a30c939a3d0d as hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b532410b94c34c58b086a30c939a3d0d
2024-11-09T21:25:45,625 INFO  [M:0;d86ff69d6719:36821 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b532410b94c34c58b086a30c939a3d0d, entries=1, sequenceid=29, filesize=5.0 K
2024-11-09T21:25:45,625 DEBUG [M:0;d86ff69d6719:36821 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6fc96ce4387f4ae595755cea2ac2eed4 as hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6fc96ce4387f4ae595755cea2ac2eed4
2024-11-09T21:25:45,629 INFO  [M:0;d86ff69d6719:36821 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45487/user/jenkins/test-data/3dd60e50-af1e-2898-5342-286db4ffc3b9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6fc96ce4387f4ae595755cea2ac2eed4, entries=1, sequenceid=29, filesize=4.9 K
2024-11-09T21:25:45,630 INFO  [M:0;d86ff69d6719:36821 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 106ms, sequenceid=29, compaction requested=false
2024-11-09T21:25:45,631 INFO  [M:0;d86ff69d6719:36821 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-09T21:25:45,631 DEBUG [M:0;d86ff69d6719:36821 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731187545524Disabling compacts and flushes for region at 1731187545524Disabling writes for close at 1731187545524Obtaining lock to block concurrent updates at 1731187545524Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731187545524Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731187545524Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731187545525 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731187545525Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731187545541 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731187545541Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731187545550 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731187545562 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731187545562Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731187545570 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731187545582 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731187545582Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731187545591 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731187545603 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731187545603Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3aadaadc: reopening flushed file at 1731187545611 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7595479d: reopening flushed file at 1731187545616 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21834c2: reopening flushed file at 1731187545620 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f574098: reopening flushed file at 1731187545625 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 106ms, sequenceid=29, compaction requested=false at 1731187545630 (+5 ms)Writing region close event to WAL at 1731187545631 (+1 ms)Closed at 1731187545631
2024-11-09T21:25:45,632 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,632 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,632 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,632 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,632 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-09T21:25:45,633 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37083 is added to blk_1073741830_1006 (size=10311)
2024-11-09T21:25:45,634 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37497 is added to blk_1073741830_1006 (size=10311)
2024-11-09T21:25:45,634 INFO  [M:0;d86ff69d6719:36821 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-09T21:25:45,634 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-09T21:25:45,634 INFO  [M:0;d86ff69d6719:36821 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36821
2024-11-09T21:25:45,635 INFO  [M:0;d86ff69d6719:36821 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-09T21:25:45,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:25:45,767 INFO  [M:0;d86ff69d6719:36821 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-09T21:25:45,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36821-0x10122c722110000, quorum=127.0.0.1:61635, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-09T21:25:45,770 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@9255255{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:25:45,770 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2869fc81{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:25:45,771 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:25:45,771 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@59dd2dec{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:25:45,771 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a513798{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/hadoop.log.dir/,STOPPED}
2024-11-09T21:25:45,773 WARN  [BP-1286790564-172.17.0.2-1731187542251 heartbeating to localhost/127.0.0.1:45487 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:25:45,773 WARN  [BP-1286790564-172.17.0.2-1731187542251 heartbeating to localhost/127.0.0.1:45487 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1286790564-172.17.0.2-1731187542251 (Datanode Uuid 04a9b58a-b44f-4e1e-b725-7e07d7e5997c) service to localhost/127.0.0.1:45487
2024-11-09T21:25:45,773 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:25:45,773 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:25:45,773 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/cluster_b81e8969-336e-2cb1-7e24-5e112673e34c/data/data3/current/BP-1286790564-172.17.0.2-1731187542251 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:25:45,774 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/cluster_b81e8969-336e-2cb1-7e24-5e112673e34c/data/data4/current/BP-1286790564-172.17.0.2-1731187542251 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:25:45,774 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:25:45,779 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@32b4a64d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-09T21:25:45,779 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2ea5aa4c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:25:45,779 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:25:45,780 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@61ab51b1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:25:45,780 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6ef29692{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/hadoop.log.dir/,STOPPED}
2024-11-09T21:25:45,781 WARN  [BP-1286790564-172.17.0.2-1731187542251 heartbeating to localhost/127.0.0.1:45487 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-09T21:25:45,781 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-09T21:25:45,781 WARN  [BP-1286790564-172.17.0.2-1731187542251 heartbeating to localhost/127.0.0.1:45487 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1286790564-172.17.0.2-1731187542251 (Datanode Uuid 913b8f79-91b0-43f9-bc81-6d2868e02ff6) service to localhost/127.0.0.1:45487
2024-11-09T21:25:45,781 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-09T21:25:45,781 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/cluster_b81e8969-336e-2cb1-7e24-5e112673e34c/data/data1/current/BP-1286790564-172.17.0.2-1731187542251 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:25:45,781 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/cluster_b81e8969-336e-2cb1-7e24-5e112673e34c/data/data2/current/BP-1286790564-172.17.0.2-1731187542251 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-09T21:25:45,782 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-09T21:25:45,787 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1773ea07{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-09T21:25:45,787 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@45235d1a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-09T21:25:45,787 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-09T21:25:45,787 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17951be7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-09T21:25:45,788 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@12ac7c6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9ff4e3ca-f94f-d97f-221a-508188a06b42/hadoop.log.dir/,STOPPED}
2024-11-09T21:25:45,794 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-09T21:25:45,806 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-09T21:25:45,814 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=272 (was 233)
Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45487
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:45487
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45487 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45487 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45487
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45487 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45487
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:45487
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=532 (was 515) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=16 (was 17), ProcessCount=11 (was 11), AvailableMemoryMB=17933 (was 17938)