2024-11-22 05:33:13,482 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-22 05:33:13,501 main DEBUG Took 0.016148 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-22 05:33:13,501 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-22 05:33:13,502 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-22 05:33:13,503 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-22 05:33:13,505 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 05:33:13,515 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-22 05:33:13,532 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 05:33:13,534 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 05:33:13,535 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 05:33:13,536 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 05:33:13,536 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 05:33:13,537 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 05:33:13,538 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 05:33:13,539 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 05:33:13,539 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 05:33:13,540 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 05:33:13,541 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 05:33:13,541 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 05:33:13,542 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 05:33:13,542 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 05:33:13,543 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 05:33:13,543 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 05:33:13,544 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 05:33:13,544 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 05:33:13,545 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 05:33:13,545 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 05:33:13,546 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 05:33:13,546 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 05:33:13,547 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 05:33:13,548 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 05:33:13,548 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 05:33:13,549 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-22 05:33:13,550 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 05:33:13,552 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-22 05:33:13,555 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-22 05:33:13,556 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-22 05:33:13,557 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-22 05:33:13,558 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-22 05:33:13,569 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-22 05:33:13,572 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-22 05:33:13,575 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-22 05:33:13,576 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-22 05:33:13,577 main DEBUG createAppenders(={Console})
2024-11-22 05:33:13,578 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-11-22 05:33:13,579 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-22 05:33:13,579 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-11-22 05:33:13,580 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-22 05:33:13,580 main DEBUG OutputStream closed
2024-11-22 05:33:13,581 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-22 05:33:13,581 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-22 05:33:13,582 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-11-22 05:33:13,695 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-22 05:33:13,699 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-22 05:33:13,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-22 05:33:13,703 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-22 05:33:13,704 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-22 05:33:13,704 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-22 05:33:13,705 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-22 05:33:13,705 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-22 05:33:13,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-22 05:33:13,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-22 05:33:13,707 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-22 05:33:13,707 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-22 05:33:13,707 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-22 05:33:13,708 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-22 05:33:13,708 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-22 05:33:13,708 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-22 05:33:13,709 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-22 05:33:13,710 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-22 05:33:13,713 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-22 05:33:13,714 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-11-22 05:33:13,714 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-22 05:33:13,715 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-11-22T05:33:14,161 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944
2024-11-22 05:33:14,165 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-22 05:33:14,165 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-22T05:33:14,178 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-11-22T05:33:14,226 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=196, ProcessCount=11, AvailableMemoryMB=8556
2024-11-22T05:33:14,231 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-22T05:33:14,254 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/cluster_986ee2ba-9b27-0ddc-1f6c-d6f5e45fb4ac, deleteOnExit=true
2024-11-22T05:33:14,254 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-22T05:33:14,256 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/test.cache.data in system properties and HBase conf
2024-11-22T05:33:14,257 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/hadoop.tmp.dir in system properties and HBase conf
2024-11-22T05:33:14,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/hadoop.log.dir in system properties and HBase conf
2024-11-22T05:33:14,260 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-22T05:33:14,261 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-22T05:33:14,262 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-22T05:33:14,366 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-22T05:33:14,485 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-22T05:33:14,491 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-22T05:33:14,492 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-22T05:33:14,493 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-22T05:33:14,494 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T05:33:14,494 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-22T05:33:14,495 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-22T05:33:14,496 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T05:33:14,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T05:33:14,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-22T05:33:14,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/nfs.dump.dir in system properties and HBase conf
2024-11-22T05:33:14,499 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/java.io.tmpdir in system properties and HBase conf
2024-11-22T05:33:14,500 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T05:33:14,500 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-22T05:33:14,501 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-22T05:33:15,048 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T05:33:15,635 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-22T05:33:15,704 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2961ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-22T05:33:15,777 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:33:15,840 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:33:15,860 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:33:15,860 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:33:15,862 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T05:33:15,874 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:33:15,877 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:33:15,878 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:33:16,054 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/java.io.tmpdir/jetty-localhost-33991-hadoop-hdfs-3_4_1-tests_jar-_-any-17448976258243920185/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T05:33:16,060 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:33991}
2024-11-22T05:33:16,060 INFO  [Time-limited test {}] server.Server(415): Started @3318ms
2024-11-22T05:33:16,092 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T05:33:16,653 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:33:16,661 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:33:16,662 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:33:16,662 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:33:16,662 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:33:16,664 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:33:16,665 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:33:16,765 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/java.io.tmpdir/jetty-localhost-40883-hadoop-hdfs-3_4_1-tests_jar-_-any-7403094487671397640/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:33:16,766 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:40883}
2024-11-22T05:33:16,766 INFO  [Time-limited test {}] server.Server(415): Started @4024ms
2024-11-22T05:33:16,814 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:33:16,916 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:33:16,921 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:33:16,922 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:33:16,922 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:33:16,922 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:33:16,923 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:33:16,924 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:33:17,034 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/java.io.tmpdir/jetty-localhost-46415-hadoop-hdfs-3_4_1-tests_jar-_-any-13097358215966288950/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:33:17,034 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:46415}
2024-11-22T05:33:17,035 INFO  [Time-limited test {}] server.Server(415): Started @4292ms
2024-11-22T05:33:17,037 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:33:18,073 WARN  [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/cluster_986ee2ba-9b27-0ddc-1f6c-d6f5e45fb4ac/data/data1/current/BP-1033121826-172.17.0.2-1732253595128/current, will proceed with Du for space computation calculation, 
2024-11-22T05:33:18,073 WARN  [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/cluster_986ee2ba-9b27-0ddc-1f6c-d6f5e45fb4ac/data/data4/current/BP-1033121826-172.17.0.2-1732253595128/current, will proceed with Du for space computation calculation, 
2024-11-22T05:33:18,073 WARN  [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/cluster_986ee2ba-9b27-0ddc-1f6c-d6f5e45fb4ac/data/data2/current/BP-1033121826-172.17.0.2-1732253595128/current, will proceed with Du for space computation calculation, 
2024-11-22T05:33:18,073 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/cluster_986ee2ba-9b27-0ddc-1f6c-d6f5e45fb4ac/data/data3/current/BP-1033121826-172.17.0.2-1732253595128/current, will proceed with Du for space computation calculation, 
2024-11-22T05:33:18,105 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:33:18,105 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:33:18,151 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944
2024-11-22T05:33:18,154 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd599960c88a2e2b9 with lease ID 0xe31109178e292b58: Processing first storage report for DS-85dcf605-e916-4551-87df-842e9e4a313c from datanode DatanodeRegistration(127.0.0.1:41279, datanodeUuid=090d0970-9579-4fee-bb82-b7d4a1142ea7, infoPort=43135, infoSecurePort=0, ipcPort=41651, storageInfo=lv=-57;cid=testClusterID;nsid=2067812557;c=1732253595128)
2024-11-22T05:33:18,155 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd599960c88a2e2b9 with lease ID 0xe31109178e292b58: from storage DS-85dcf605-e916-4551-87df-842e9e4a313c node DatanodeRegistration(127.0.0.1:41279, datanodeUuid=090d0970-9579-4fee-bb82-b7d4a1142ea7, infoPort=43135, infoSecurePort=0, ipcPort=41651, storageInfo=lv=-57;cid=testClusterID;nsid=2067812557;c=1732253595128), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-11-22T05:33:18,156 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xea135d1cc0bb8135 with lease ID 0xe31109178e292b57: Processing first storage report for DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca from datanode DatanodeRegistration(127.0.0.1:37373, datanodeUuid=7f3bcfa2-3469-4bda-8d6c-03316cb48ec7, infoPort=38455, infoSecurePort=0, ipcPort=37445, storageInfo=lv=-57;cid=testClusterID;nsid=2067812557;c=1732253595128)
2024-11-22T05:33:18,156 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xea135d1cc0bb8135 with lease ID 0xe31109178e292b57: from storage DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca node DatanodeRegistration(127.0.0.1:37373, datanodeUuid=7f3bcfa2-3469-4bda-8d6c-03316cb48ec7, infoPort=38455, infoSecurePort=0, ipcPort=37445, storageInfo=lv=-57;cid=testClusterID;nsid=2067812557;c=1732253595128), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-22T05:33:18,157 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd599960c88a2e2b9 with lease ID 0xe31109178e292b58: Processing first storage report for DS-525b5288-7723-4b77-84cb-79747871ca52 from datanode DatanodeRegistration(127.0.0.1:41279, datanodeUuid=090d0970-9579-4fee-bb82-b7d4a1142ea7, infoPort=43135, infoSecurePort=0, ipcPort=41651, storageInfo=lv=-57;cid=testClusterID;nsid=2067812557;c=1732253595128)
2024-11-22T05:33:18,157 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd599960c88a2e2b9 with lease ID 0xe31109178e292b58: from storage DS-525b5288-7723-4b77-84cb-79747871ca52 node DatanodeRegistration(127.0.0.1:41279, datanodeUuid=090d0970-9579-4fee-bb82-b7d4a1142ea7, infoPort=43135, infoSecurePort=0, ipcPort=41651, storageInfo=lv=-57;cid=testClusterID;nsid=2067812557;c=1732253595128), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:33:18,157 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xea135d1cc0bb8135 with lease ID 0xe31109178e292b57: Processing first storage report for DS-24c553de-8e1a-40d6-bb8c-aaab00600783 from datanode DatanodeRegistration(127.0.0.1:37373, datanodeUuid=7f3bcfa2-3469-4bda-8d6c-03316cb48ec7, infoPort=38455, infoSecurePort=0, ipcPort=37445, storageInfo=lv=-57;cid=testClusterID;nsid=2067812557;c=1732253595128)
2024-11-22T05:33:18,157 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xea135d1cc0bb8135 with lease ID 0xe31109178e292b57: from storage DS-24c553de-8e1a-40d6-bb8c-aaab00600783 node DatanodeRegistration(127.0.0.1:37373, datanodeUuid=7f3bcfa2-3469-4bda-8d6c-03316cb48ec7, infoPort=38455, infoSecurePort=0, ipcPort=37445, storageInfo=lv=-57;cid=testClusterID;nsid=2067812557;c=1732253595128), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:33:18,217 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/cluster_986ee2ba-9b27-0ddc-1f6c-d6f5e45fb4ac/zookeeper_0, clientPort=54019, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/cluster_986ee2ba-9b27-0ddc-1f6c-d6f5e45fb4ac/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/cluster_986ee2ba-9b27-0ddc-1f6c-d6f5e45fb4ac/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-22T05:33:18,225 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54019
2024-11-22T05:33:18,235 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:33:18,237 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:33:18,445 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741825_1001 (size=7)
2024-11-22T05:33:18,446 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741825_1001 (size=7)
2024-11-22T05:33:18,857 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b with version=8
2024-11-22T05:33:18,858 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/hbase-staging
2024-11-22T05:33:18,934 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-22T05:33:19,248 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/82891a793a29:0 server-side Connection retries=45
2024-11-22T05:33:19,264 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:33:19,265 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T05:33:19,271 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T05:33:19,272 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:33:19,272 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T05:33:19,428 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-22T05:33:19,495 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-22T05:33:19,505 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-22T05:33:19,508 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T05:33:19,542 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 99023 (auto-detected)
2024-11-22T05:33:19,544 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-22T05:33:19,571 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43561
2024-11-22T05:33:19,597 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43561 connecting to ZooKeeper ensemble=127.0.0.1:54019
2024-11-22T05:33:19,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:435610x0, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T05:33:19,724 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43561-0x10161083da50000 connected
2024-11-22T05:33:19,813 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:33:19,816 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:33:19,825 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:33:19,829 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b, hbase.cluster.distributed=false
2024-11-22T05:33:19,850 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T05:33:19,854 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43561
2024-11-22T05:33:19,854 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43561
2024-11-22T05:33:19,854 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43561
2024-11-22T05:33:19,855 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43561
2024-11-22T05:33:19,855 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43561
2024-11-22T05:33:19,945 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/82891a793a29:0 server-side Connection retries=45
2024-11-22T05:33:19,946 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:33:19,946 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T05:33:19,947 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T05:33:19,947 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:33:19,947 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T05:33:19,949 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-22T05:33:19,952 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T05:33:19,953 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42405
2024-11-22T05:33:19,954 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42405 connecting to ZooKeeper ensemble=127.0.0.1:54019
2024-11-22T05:33:19,955 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:33:19,959 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:33:19,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:424050x0, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T05:33:19,976 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:424050x0, quorum=127.0.0.1:54019, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:33:19,976 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42405-0x10161083da50001 connected
2024-11-22T05:33:19,980 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-22T05:33:19,988 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-22T05:33:19,990 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-22T05:33:19,995 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T05:33:19,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42405
2024-11-22T05:33:19,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42405
2024-11-22T05:33:19,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42405
2024-11-22T05:33:19,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42405
2024-11-22T05:33:19,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42405
2024-11-22T05:33:20,012 DEBUG [M:0;82891a793a29:43561 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;82891a793a29:43561
2024-11-22T05:33:20,013 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/82891a793a29,43561,1732253599023
2024-11-22T05:33:20,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:33:20,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:33:20,089 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/82891a793a29,43561,1732253599023
2024-11-22T05:33:20,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-22T05:33:20,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:33:20,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:33:20,134 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-22T05:33:20,136 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/82891a793a29,43561,1732253599023 from backup master directory
2024-11-22T05:33:20,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/82891a793a29,43561,1732253599023
2024-11-22T05:33:20,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:33:20,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:33:20,149 WARN  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T05:33:20,149 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=82891a793a29,43561,1732253599023
2024-11-22T05:33:20,152 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-22T05:33:20,154 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-22T05:33:20,206 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/hbase.id] with ID: 9ea13ed1-0ca3-4112-9d48-9218df481f6e
2024-11-22T05:33:20,206 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/.tmp/hbase.id
2024-11-22T05:33:20,218 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741826_1002 (size=42)
2024-11-22T05:33:20,219 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741826_1002 (size=42)
2024-11-22T05:33:20,220 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/.tmp/hbase.id]:[hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/hbase.id]
2024-11-22T05:33:20,261 INFO  [master/82891a793a29:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:33:20,266 INFO  [master/82891a793a29:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-22T05:33:20,285 INFO  [master/82891a793a29:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms.
2024-11-22T05:33:20,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:33:20,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:33:20,312 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741827_1003 (size=196)
2024-11-22T05:33:20,312 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741827_1003 (size=196)
2024-11-22T05:33:20,327 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T05:33:20,328 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-22T05:33:20,333 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:33:20,367 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741828_1004 (size=1189)
2024-11-22T05:33:20,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741828_1004 (size=1189)
2024-11-22T05:33:20,383 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store
2024-11-22T05:33:20,401 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741829_1005 (size=34)
2024-11-22T05:33:20,401 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741829_1005 (size=34)
2024-11-22T05:33:20,408 INFO  [master/82891a793a29:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-22T05:33:20,412 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:33:20,414 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T05:33:20,414 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:33:20,414 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:33:20,416 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T05:33:20,417 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:33:20,417 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:33:20,418 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732253600414Disabling compacts and flushes for region at 1732253600414Disabling writes for close at 1732253600417 (+3 ms)Writing region close event to WAL at 1732253600417Closed at 1732253600417
2024-11-22T05:33:20,420 WARN  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/.initializing
2024-11-22T05:33:20,421 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/WALs/82891a793a29,43561,1732253599023
2024-11-22T05:33:20,443 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C43561%2C1732253599023, suffix=, logDir=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/WALs/82891a793a29,43561,1732253599023, archiveDir=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/oldWALs, maxLogs=10
2024-11-22T05:33:20,454 INFO  [master/82891a793a29:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C43561%2C1732253599023.1732253600449
2024-11-22T05:33:20,474 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/WALs/82891a793a29,43561,1732253599023/82891a793a29%2C43561%2C1732253599023.1732253600449
2024-11-22T05:33:20,481 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38455:38455),(127.0.0.1/127.0.0.1:43135:43135)]
2024-11-22T05:33:20,482 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:33:20,483 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:33:20,485 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:33:20,486 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:33:20,519 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:33:20,542 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-22T05:33:20,545 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:33:20,547 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:33:20,548 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:33:20,552 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-22T05:33:20,552 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:33:20,553 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:33:20,554 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:33:20,557 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-22T05:33:20,557 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:33:20,558 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:33:20,558 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:33:20,561 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-22T05:33:20,561 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:33:20,562 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:33:20,563 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:33:20,567 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:33:20,568 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:33:20,574 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:33:20,575 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:33:20,579 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-22T05:33:20,583 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:33:20,587 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:33:20,588 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=835201, jitterRate=0.06201407313346863}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-22T05:33:20,595 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732253600497Initializing all the Stores at 1732253600499 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253600499Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253600500 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253600500Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253600500Cleaning up temporary data from old regions at 1732253600575 (+75 ms)Region opened successfully at 1732253600595 (+20 ms)
2024-11-22T05:33:20,597 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-22T05:33:20,627 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2800d548, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82891a793a29/172.17.0.2:0
2024-11-22T05:33:20,653 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-22T05:33:20,662 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-22T05:33:20,662 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-22T05:33:20,665 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-22T05:33:20,666 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-22T05:33:20,670 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-22T05:33:20,670 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-22T05:33:20,694 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-22T05:33:20,701 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-22T05:33:20,733 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-22T05:33:20,737 INFO  [master/82891a793a29:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-22T05:33:20,740 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-22T05:33:20,822 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-22T05:33:20,828 INFO  [master/82891a793a29:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-22T05:33:20,832 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-22T05:33:20,842 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-22T05:33:20,845 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-22T05:33:20,853 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-22T05:33:20,873 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-22T05:33:20,884 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-22T05:33:20,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T05:33:20,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T05:33:20,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:33:20,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:33:20,900 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=82891a793a29,43561,1732253599023, sessionid=0x10161083da50000, setting cluster-up flag (Was=false)
2024-11-22T05:33:20,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:33:20,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:33:20,969 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-22T05:33:20,973 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=82891a793a29,43561,1732253599023
2024-11-22T05:33:20,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:33:20,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:33:21,032 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-22T05:33:21,034 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=82891a793a29,43561,1732253599023
2024-11-22T05:33:21,040 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-22T05:33:21,102 INFO  [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(746): ClusterId : 9ea13ed1-0ca3-4112-9d48-9218df481f6e
2024-11-22T05:33:21,104 DEBUG [RS:0;82891a793a29:42405 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-22T05:33:21,105 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-22T05:33:21,115 INFO  [master/82891a793a29:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-22T05:33:21,118 DEBUG [RS:0;82891a793a29:42405 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-22T05:33:21,118 DEBUG [RS:0;82891a793a29:42405 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-22T05:33:21,122 INFO  [master/82891a793a29:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-22T05:33:21,128 DEBUG [RS:0;82891a793a29:42405 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-22T05:33:21,129 DEBUG [RS:0;82891a793a29:42405 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1190ca78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82891a793a29/172.17.0.2:0
2024-11-22T05:33:21,128 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 82891a793a29,43561,1732253599023
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-22T05:33:21,135 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:33:21,135 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:33:21,136 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:33:21,136 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:33:21,136 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/82891a793a29:0, corePoolSize=10, maxPoolSize=10
2024-11-22T05:33:21,136 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:33:21,136 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/82891a793a29:0, corePoolSize=2, maxPoolSize=2
2024-11-22T05:33:21,136 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:33:21,140 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732253631140
2024-11-22T05:33:21,142 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:33:21,142 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-22T05:33:21,142 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-22T05:33:21,143 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-22T05:33:21,147 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-22T05:33:21,147 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-22T05:33:21,147 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-22T05:33:21,148 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-22T05:33:21,148 DEBUG [RS:0;82891a793a29:42405 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;82891a793a29:42405
2024-11-22T05:33:21,150 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:33:21,148 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:21,150 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-22T05:33:21,152 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-22T05:33:21,153 INFO  [RS:0;82891a793a29:42405 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-22T05:33:21,153 INFO  [RS:0;82891a793a29:42405 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-22T05:33:21,153 DEBUG [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-22T05:33:21,154 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-22T05:33:21,154 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-22T05:33:21,156 INFO  [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(2659): reportForDuty to master=82891a793a29,43561,1732253599023 with port=42405, startcode=1732253599913
2024-11-22T05:33:21,157 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-22T05:33:21,157 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-22T05:33:21,163 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253601158,5,FailOnTimeoutGroup]
2024-11-22T05:33:21,164 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253601164,5,FailOnTimeoutGroup]
2024-11-22T05:33:21,165 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:21,165 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-22T05:33:21,166 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:21,167 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:21,167 DEBUG [RS:0;82891a793a29:42405 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-22T05:33:21,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741831_1007 (size=1321)
2024-11-22T05:33:21,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741831_1007 (size=1321)
2024-11-22T05:33:21,172 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-22T05:33:21,173 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b
2024-11-22T05:33:21,187 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741832_1008 (size=32)
2024-11-22T05:33:21,187 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741832_1008 (size=32)
2024-11-22T05:33:21,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:33:21,193 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T05:33:21,196 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T05:33:21,196 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:33:21,197 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:33:21,198 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T05:33:21,201 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T05:33:21,201 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:33:21,202 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:33:21,203 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T05:33:21,206 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T05:33:21,207 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:33:21,208 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:33:21,208 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T05:33:21,214 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T05:33:21,214 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:33:21,215 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:33:21,216 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T05:33:21,218 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740
2024-11-22T05:33:21,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740
2024-11-22T05:33:21,223 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T05:33:21,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T05:33:21,225 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T05:33:21,229 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T05:33:21,237 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:33:21,238 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=713941, jitterRate=-0.09217788279056549}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T05:33:21,242 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732253601190Initializing all the Stores at 1732253601192 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253601192Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253601192Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253601192Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253601192Cleaning up temporary data from old regions at 1732253601224 (+32 ms)Region opened successfully at 1732253601242 (+18 ms)
2024-11-22T05:33:21,243 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T05:33:21,243 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T05:33:21,243 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T05:33:21,243 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T05:33:21,243 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T05:33:21,244 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T05:33:21,244 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49813, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-22T05:33:21,244 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732253601243Disabling compacts and flushes for region at 1732253601243Disabling writes for close at 1732253601243Writing region close event to WAL at 1732253601244 (+1 ms)Closed at 1732253601244
2024-11-22T05:33:21,248 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:33:21,248 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-22T05:33:21,253 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43561 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 82891a793a29,42405,1732253599913
2024-11-22T05:33:21,256 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43561 {}] master.ServerManager(517): Registering regionserver=82891a793a29,42405,1732253599913
2024-11-22T05:33:21,256 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-22T05:33:21,267 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T05:33:21,269 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-22T05:33:21,272 DEBUG [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b
2024-11-22T05:33:21,272 DEBUG [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40545
2024-11-22T05:33:21,272 DEBUG [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-22T05:33:21,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T05:33:21,285 DEBUG [RS:0;82891a793a29:42405 {}] zookeeper.ZKUtil(111): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/82891a793a29,42405,1732253599913
2024-11-22T05:33:21,285 WARN  [RS:0;82891a793a29:42405 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T05:33:21,286 INFO  [RS:0;82891a793a29:42405 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:33:21,286 DEBUG [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913
2024-11-22T05:33:21,288 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [82891a793a29,42405,1732253599913]
2024-11-22T05:33:21,310 INFO  [RS:0;82891a793a29:42405 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-22T05:33:21,326 INFO  [RS:0;82891a793a29:42405 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-22T05:33:21,330 INFO  [RS:0;82891a793a29:42405 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T05:33:21,331 INFO  [RS:0;82891a793a29:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:21,332 INFO  [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-22T05:33:21,337 INFO  [RS:0;82891a793a29:42405 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-22T05:33:21,338 INFO  [RS:0;82891a793a29:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:21,339 DEBUG [RS:0;82891a793a29:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:33:21,339 DEBUG [RS:0;82891a793a29:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:33:21,339 DEBUG [RS:0;82891a793a29:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:33:21,339 DEBUG [RS:0;82891a793a29:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:33:21,339 DEBUG [RS:0;82891a793a29:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:33:21,339 DEBUG [RS:0;82891a793a29:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/82891a793a29:0, corePoolSize=2, maxPoolSize=2
2024-11-22T05:33:21,339 DEBUG [RS:0;82891a793a29:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:33:21,339 DEBUG [RS:0;82891a793a29:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:33:21,340 DEBUG [RS:0;82891a793a29:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:33:21,340 DEBUG [RS:0;82891a793a29:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:33:21,340 DEBUG [RS:0;82891a793a29:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:33:21,340 DEBUG [RS:0;82891a793a29:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:33:21,340 DEBUG [RS:0;82891a793a29:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/82891a793a29:0, corePoolSize=3, maxPoolSize=3
2024-11-22T05:33:21,340 DEBUG [RS:0;82891a793a29:42405 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0, corePoolSize=3, maxPoolSize=3
2024-11-22T05:33:21,341 INFO  [RS:0;82891a793a29:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:21,341 INFO  [RS:0;82891a793a29:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:21,342 INFO  [RS:0;82891a793a29:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:21,342 INFO  [RS:0;82891a793a29:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:21,342 INFO  [RS:0;82891a793a29:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:21,342 INFO  [RS:0;82891a793a29:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,42405,1732253599913-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T05:33:21,358 INFO  [RS:0;82891a793a29:42405 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-22T05:33:21,360 INFO  [RS:0;82891a793a29:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,42405,1732253599913-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:21,360 INFO  [RS:0;82891a793a29:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:21,361 INFO  [RS:0;82891a793a29:42405 {}] regionserver.Replication(171): 82891a793a29,42405,1732253599913 started
2024-11-22T05:33:21,378 INFO  [RS:0;82891a793a29:42405 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:21,379 INFO  [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(1482): Serving as 82891a793a29,42405,1732253599913, RpcServer on 82891a793a29/172.17.0.2:42405, sessionid=0x10161083da50001
2024-11-22T05:33:21,380 DEBUG [RS:0;82891a793a29:42405 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-22T05:33:21,380 DEBUG [RS:0;82891a793a29:42405 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 82891a793a29,42405,1732253599913
2024-11-22T05:33:21,380 DEBUG [RS:0;82891a793a29:42405 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82891a793a29,42405,1732253599913'
2024-11-22T05:33:21,380 DEBUG [RS:0;82891a793a29:42405 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-22T05:33:21,381 DEBUG [RS:0;82891a793a29:42405 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-22T05:33:21,382 DEBUG [RS:0;82891a793a29:42405 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-22T05:33:21,382 DEBUG [RS:0;82891a793a29:42405 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-22T05:33:21,382 DEBUG [RS:0;82891a793a29:42405 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 82891a793a29,42405,1732253599913
2024-11-22T05:33:21,382 DEBUG [RS:0;82891a793a29:42405 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82891a793a29,42405,1732253599913'
2024-11-22T05:33:21,383 DEBUG [RS:0;82891a793a29:42405 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-22T05:33:21,383 DEBUG [RS:0;82891a793a29:42405 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-22T05:33:21,384 DEBUG [RS:0;82891a793a29:42405 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-22T05:33:21,384 INFO  [RS:0;82891a793a29:42405 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-22T05:33:21,384 INFO  [RS:0;82891a793a29:42405 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-22T05:33:21,420 WARN  [82891a793a29:43561 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-22T05:33:21,496 INFO  [RS:0;82891a793a29:42405 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C42405%2C1732253599913, suffix=, logDir=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913, archiveDir=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/oldWALs, maxLogs=32
2024-11-22T05:33:21,498 INFO  [RS:0;82891a793a29:42405 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42405%2C1732253599913.1732253601498
2024-11-22T05:33:21,507 INFO  [RS:0;82891a793a29:42405 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253601498
2024-11-22T05:33:21,512 DEBUG [RS:0;82891a793a29:42405 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43135:43135),(127.0.0.1/127.0.0.1:38455:38455)]
2024-11-22T05:33:21,674 DEBUG [82891a793a29:43561 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-22T05:33:21,688 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=82891a793a29,42405,1732253599913
2024-11-22T05:33:21,693 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 82891a793a29,42405,1732253599913, state=OPENING
2024-11-22T05:33:21,727 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-22T05:33:21,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:33:21,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:33:21,815 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:33:21,815 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:33:21,819 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T05:33:21,822 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=82891a793a29,42405,1732253599913}]
2024-11-22T05:33:22,004 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-22T05:33:22,008 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45789, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-22T05:33:22,019 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-22T05:33:22,020 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:33:22,023 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C42405%2C1732253599913.meta, suffix=.meta, logDir=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913, archiveDir=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/oldWALs, maxLogs=32
2024-11-22T05:33:22,025 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42405%2C1732253599913.meta.1732253602025.meta
2024-11-22T05:33:22,032 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.meta.1732253602025.meta
2024-11-22T05:33:22,037 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43135:43135),(127.0.0.1/127.0.0.1:38455:38455)]
2024-11-22T05:33:22,039 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:33:22,041 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-22T05:33:22,044 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-22T05:33:22,048 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-22T05:33:22,053 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-22T05:33:22,053 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:33:22,054 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-22T05:33:22,054 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-22T05:33:22,057 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T05:33:22,059 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T05:33:22,059 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:33:22,059 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:33:22,060 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T05:33:22,061 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T05:33:22,061 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:33:22,062 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:33:22,063 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T05:33:22,064 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T05:33:22,064 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:33:22,065 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:33:22,065 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T05:33:22,066 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T05:33:22,067 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:33:22,067 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:33:22,068 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T05:33:22,069 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740
2024-11-22T05:33:22,072 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740
2024-11-22T05:33:22,074 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T05:33:22,074 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T05:33:22,075 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T05:33:22,078 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T05:33:22,079 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=873805, jitterRate=0.11110176146030426}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T05:33:22,079 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-22T05:33:22,081 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732253602055Writing region info on filesystem at 1732253602055Initializing all the Stores at 1732253602056 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253602056Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253602057 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253602057Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253602057Cleaning up temporary data from old regions at 1732253602074 (+17 ms)Running coprocessor post-open hooks at 1732253602079 (+5 ms)Region opened successfully at 1732253602080 (+1 ms)
2024-11-22T05:33:22,086 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732253601994
2024-11-22T05:33:22,098 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-22T05:33:22,098 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-22T05:33:22,100 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=82891a793a29,42405,1732253599913
2024-11-22T05:33:22,102 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 82891a793a29,42405,1732253599913, state=OPEN
2024-11-22T05:33:22,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T05:33:22,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T05:33:22,148 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:33:22,148 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:33:22,148 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=82891a793a29,42405,1732253599913
2024-11-22T05:33:22,154 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-22T05:33:22,155 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=82891a793a29,42405,1732253599913 in 326 msec
2024-11-22T05:33:22,162 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-22T05:33:22,162 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 900 msec
2024-11-22T05:33:22,164 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:33:22,164 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-22T05:33:22,181 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T05:33:22,182 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82891a793a29,42405,1732253599913, seqNum=-1]
2024-11-22T05:33:22,203 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T05:33:22,205 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38335, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T05:33:22,224 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1550 sec
2024-11-22T05:33:22,224 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732253602224, completionTime=-1
2024-11-22T05:33:22,227 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-22T05:33:22,228 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-22T05:33:22,253 INFO  [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-22T05:33:22,253 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732253662253
2024-11-22T05:33:22,253 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732253722253
2024-11-22T05:33:22,253 INFO  [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec
2024-11-22T05:33:22,256 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,43561,1732253599023-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:22,257 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,43561,1732253599023-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:22,257 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,43561,1732253599023-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:22,258 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-82891a793a29:43561, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:22,258 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:22,259 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:22,265 DEBUG [master/82891a793a29:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-22T05:33:22,290 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.141sec
2024-11-22T05:33:22,291 INFO  [master/82891a793a29:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-22T05:33:22,292 INFO  [master/82891a793a29:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-22T05:33:22,293 INFO  [master/82891a793a29:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-22T05:33:22,293 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-22T05:33:22,293 INFO  [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-22T05:33:22,294 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,43561,1732253599023-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T05:33:22,294 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,43561,1732253599023-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-22T05:33:22,302 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-22T05:33:22,303 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-22T05:33:22,304 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,43561,1732253599023-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:33:22,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66cb686a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:33:22,317 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-22T05:33:22,317 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-22T05:33:22,321 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82891a793a29,43561,-1 for getting cluster id
2024-11-22T05:33:22,324 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-22T05:33:22,332 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9ea13ed1-0ca3-4112-9d48-9218df481f6e'
2024-11-22T05:33:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-22T05:33:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9ea13ed1-0ca3-4112-9d48-9218df481f6e"

2024-11-22T05:33:22,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38b0dd74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:33:22,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82891a793a29,43561,-1]
2024-11-22T05:33:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-22T05:33:22,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:33:22,344 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-22T05:33:22,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32fb6022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:33:22,348 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T05:33:22,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82891a793a29,42405,1732253599913, seqNum=-1]
2024-11-22T05:33:22,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T05:33:22,360 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T05:33:22,380 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=82891a793a29,43561,1732253599023
2024-11-22T05:33:22,381 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:33:22,388 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-22T05:33:22,391 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-22T05:33:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 82891a793a29,43561,1732253599023
2024-11-22T05:33:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4363297a
2024-11-22T05:33:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-22T05:33:22,423 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-22T05:33:22,426 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43561 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-22T05:33:22,426 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43561 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-22T05:33:22,431 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43561 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T05:33:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43561 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-11-22T05:33:22,448 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-22T05:33:22,450 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43561 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-11-22T05:33:22,451 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:33:22,453 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-22T05:33:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43561 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T05:33:22,492 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741835_1011 (size=389)
2024-11-22T05:33:22,492 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741835_1011 (size=389)
2024-11-22T05:33:22,495 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1efc7679337ac59f43f1b3f9a6d90462, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b
2024-11-22T05:33:22,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741836_1012 (size=72)
2024-11-22T05:33:22,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741836_1012 (size=72)
2024-11-22T05:33:22,910 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:33:22,911 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 1efc7679337ac59f43f1b3f9a6d90462, disabling compactions & flushes
2024-11-22T05:33:22,911 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.
2024-11-22T05:33:22,911 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.
2024-11-22T05:33:22,911 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462. after waiting 0 ms
2024-11-22T05:33:22,911 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.
2024-11-22T05:33:22,911 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.
2024-11-22T05:33:22,912 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1efc7679337ac59f43f1b3f9a6d90462:
Waiting for close lock at 1732253602910Disabling compacts and flushes for region at 1732253602910Disabling writes for close at 1732253602911 (+1 ms)Writing region close event to WAL at 1732253602911Closed at 1732253602911
2024-11-22T05:33:22,915 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-22T05:33:22,921 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1732253602915"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732253602915"}]},"ts":"1732253602915"}
2024-11-22T05:33:22,926 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-22T05:33:22,928 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-22T05:33:22,931 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732253602928"}]},"ts":"1732253602928"}
2024-11-22T05:33:22,935 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-11-22T05:33:22,937 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=1efc7679337ac59f43f1b3f9a6d90462, ASSIGN}]
2024-11-22T05:33:22,940 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=1efc7679337ac59f43f1b3f9a6d90462, ASSIGN
2024-11-22T05:33:22,942 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=1efc7679337ac59f43f1b3f9a6d90462, ASSIGN; state=OFFLINE, location=82891a793a29,42405,1732253599913; forceNewPlan=false, retain=false
2024-11-22T05:33:23,094 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1efc7679337ac59f43f1b3f9a6d90462, regionState=OPENING, regionLocation=82891a793a29,42405,1732253599913
2024-11-22T05:33:23,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=1efc7679337ac59f43f1b3f9a6d90462, ASSIGN because future has completed
2024-11-22T05:33:23,101 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1efc7679337ac59f43f1b3f9a6d90462, server=82891a793a29,42405,1732253599913}]
2024-11-22T05:33:23,266 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.
2024-11-22T05:33:23,266 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1efc7679337ac59f43f1b3f9a6d90462, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:33:23,267 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 1efc7679337ac59f43f1b3f9a6d90462
2024-11-22T05:33:23,267 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:33:23,268 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1efc7679337ac59f43f1b3f9a6d90462
2024-11-22T05:33:23,268 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1efc7679337ac59f43f1b3f9a6d90462
2024-11-22T05:33:23,271 INFO  [StoreOpener-1efc7679337ac59f43f1b3f9a6d90462-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1efc7679337ac59f43f1b3f9a6d90462 
2024-11-22T05:33:23,274 INFO  [StoreOpener-1efc7679337ac59f43f1b3f9a6d90462-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1efc7679337ac59f43f1b3f9a6d90462 columnFamilyName info
2024-11-22T05:33:23,274 DEBUG [StoreOpener-1efc7679337ac59f43f1b3f9a6d90462-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:33:23,275 INFO  [StoreOpener-1efc7679337ac59f43f1b3f9a6d90462-1 {}] regionserver.HStore(327): Store=1efc7679337ac59f43f1b3f9a6d90462/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:33:23,275 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1efc7679337ac59f43f1b3f9a6d90462
2024-11-22T05:33:23,277 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462
2024-11-22T05:33:23,277 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462
2024-11-22T05:33:23,278 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1efc7679337ac59f43f1b3f9a6d90462
2024-11-22T05:33:23,278 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1efc7679337ac59f43f1b3f9a6d90462
2024-11-22T05:33:23,281 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1efc7679337ac59f43f1b3f9a6d90462
2024-11-22T05:33:23,284 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:33:23,285 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1efc7679337ac59f43f1b3f9a6d90462; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=769761, jitterRate=-0.02119871973991394}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-22T05:33:23,285 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1efc7679337ac59f43f1b3f9a6d90462
2024-11-22T05:33:23,286 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1efc7679337ac59f43f1b3f9a6d90462:
Running coprocessor pre-open hook at 1732253603268Writing region info on filesystem at 1732253603268Initializing all the Stores at 1732253603271 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253603271Cleaning up temporary data from old regions at 1732253603278 (+7 ms)Running coprocessor post-open hooks at 1732253603285 (+7 ms)Region opened successfully at 1732253603286 (+1 ms)
2024-11-22T05:33:23,288 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462., pid=6, masterSystemTime=1732253603257
2024-11-22T05:33:23,292 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.
2024-11-22T05:33:23,292 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.
2024-11-22T05:33:23,293 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1efc7679337ac59f43f1b3f9a6d90462, regionState=OPEN, openSeqNum=2, regionLocation=82891a793a29,42405,1732253599913
2024-11-22T05:33:23,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1efc7679337ac59f43f1b3f9a6d90462, server=82891a793a29,42405,1732253599913 because future has completed
2024-11-22T05:33:23,302 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-22T05:33:23,302 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1efc7679337ac59f43f1b3f9a6d90462, server=82891a793a29,42405,1732253599913 in 197 msec
2024-11-22T05:33:23,306 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-22T05:33:23,306 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=1efc7679337ac59f43f1b3f9a6d90462, ASSIGN in 365 msec
2024-11-22T05:33:23,307 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-22T05:33:23,308 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732253603307"}]},"ts":"1732253603307"}
2024-11-22T05:33:23,311 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-11-22T05:33:23,314 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-22T05:33:23,318 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 879 msec
2024-11-22T05:33:27,563 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-22T05:33:27,619 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-22T05:33:27,621 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-11-22T05:33:29,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-22T05:33:29,492 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-22T05:33:29,494 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-22T05:33:29,494 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-22T05:33:29,494 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T05:33:29,495 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-22T05:33:29,495 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-22T05:33:29,495 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-22T05:33:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43561 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T05:33:32,554 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-11-22T05:33:32,557 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-11-22T05:33:32,566 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-11-22T05:33:32,568 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.
2024-11-22T05:33:32,569 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42405%2C1732253599913.1732253612569
2024-11-22T05:33:32,578 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:33:32,578 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:33:32,578 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:33:32,578 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:33:32,578 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:33:32,579 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253601498 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253612569
2024-11-22T05:33:32,582 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741833_1009 (size=451)
2024-11-22T05:33:32,583 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741833_1009 (size=451)
2024-11-22T05:33:32,589 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253601498 to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/oldWALs/82891a793a29%2C42405%2C1732253599913.1732253601498
2024-11-22T05:33:32,590 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38455:38455),(127.0.0.1/127.0.0.1:43135:43135)]
2024-11-22T05:33:32,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462., hostname=82891a793a29,42405,1732253599913, seqNum=2]
2024-11-22T05:33:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42405 {}] regionserver.HRegion(8855): Flush requested on 1efc7679337ac59f43f1b3f9a6d90462
2024-11-22T05:33:44,642 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1efc7679337ac59f43f1b3f9a6d90462 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T05:33:44,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/d2217d29dc7342daab521b589a2c8d40 is 1080, key is row0001/info:/1732253612604/Put/seqid=0
2024-11-22T05:33:44,732 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741838_1014 (size=12509)
2024-11-22T05:33:44,733 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741838_1014 (size=12509)
2024-11-22T05:33:44,734 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/d2217d29dc7342daab521b589a2c8d40
2024-11-22T05:33:44,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/d2217d29dc7342daab521b589a2c8d40 as hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/d2217d29dc7342daab521b589a2c8d40
2024-11-22T05:33:44,817 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/d2217d29dc7342daab521b589a2c8d40, entries=7, sequenceid=11, filesize=12.2 K
2024-11-22T05:33:44,826 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 1efc7679337ac59f43f1b3f9a6d90462 in 183ms, sequenceid=11, compaction requested=false
2024-11-22T05:33:44,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1efc7679337ac59f43f1b3f9a6d90462:

2024-11-22T05:33:48,148 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-22T05:33:52,655 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42405%2C1732253599913.1732253632654
2024-11-22T05:33:52,873 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 213 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:33:52,874 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:33:52,874 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:33:52,874 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:33:52,874 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:33:52,874 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:33:52,875 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253612569 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253632654
2024-11-22T05:33:52,876 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43135:43135),(127.0.0.1/127.0.0.1:38455:38455)]
2024-11-22T05:33:52,876 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253612569 is not closed yet, will try archiving it next time
2024-11-22T05:33:52,877 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741837_1013 (size=12399)
2024-11-22T05:33:52,878 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741837_1013 (size=12399)
2024-11-22T05:33:53,081 INFO  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK], DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK]]
2024-11-22T05:33:55,290 INFO  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK], DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK]]
2024-11-22T05:33:57,496 INFO  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK], DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK]]
2024-11-22T05:33:59,700 INFO  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK], DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK]]
2024-11-22T05:33:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42405 {}] regionserver.HRegion(8855): Flush requested on 1efc7679337ac59f43f1b3f9a6d90462
2024-11-22T05:33:59,701 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1efc7679337ac59f43f1b3f9a6d90462 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T05:33:59,903 INFO  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK], DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK]]
2024-11-22T05:33:59,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/95524d1bb75a451185ad65fbc3d29bde is 1080, key is row0008/info:/1732253626641/Put/seqid=0
2024-11-22T05:33:59,922 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741840_1016 (size=12509)
2024-11-22T05:33:59,922 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741840_1016 (size=12509)
2024-11-22T05:33:59,924 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/95524d1bb75a451185ad65fbc3d29bde
2024-11-22T05:33:59,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/95524d1bb75a451185ad65fbc3d29bde as hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/95524d1bb75a451185ad65fbc3d29bde
2024-11-22T05:33:59,962 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/95524d1bb75a451185ad65fbc3d29bde, entries=7, sequenceid=21, filesize=12.2 K
2024-11-22T05:34:00,164 INFO  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK], DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK]]
2024-11-22T05:34:00,165 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 1efc7679337ac59f43f1b3f9a6d90462 in 463ms, sequenceid=21, compaction requested=false
2024-11-22T05:34:00,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1efc7679337ac59f43f1b3f9a6d90462:

2024-11-22T05:34:00,165 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-11-22T05:34:00,165 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:34:00,166 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/d2217d29dc7342daab521b589a2c8d40 because midkey is the same as first or last row
2024-11-22T05:34:01,906 INFO  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK], DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK]]
2024-11-22T05:34:02,305 INFO  [master/82891a793a29:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-22T05:34:02,305 INFO  [master/82891a793a29:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-22T05:34:04,115 INFO  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK], DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK]]
2024-11-22T05:34:04,120 WARN  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK], DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK]]
2024-11-22T05:34:04,123 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 82891a793a29%2C42405%2C1732253599913:(num 1732253632654) roll requested
2024-11-22T05:34:04,124 INFO  [regionserver/82891a793a29:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42405%2C1732253599913.1732253644124
2024-11-22T05:34:04,337 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 210 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK], DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK]]
2024-11-22T05:34:04,337 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:04,337 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:04,338 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:04,338 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:04,338 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:04,338 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253632654 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253644124
2024-11-22T05:34:04,340 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38455:38455),(127.0.0.1/127.0.0.1:43135:43135)]
2024-11-22T05:34:04,341 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253632654 is not closed yet, will try archiving it next time
2024-11-22T05:34:04,341 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253612569 to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/oldWALs/82891a793a29%2C42405%2C1732253599913.1732253612569
2024-11-22T05:34:04,342 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741839_1015 (size=7739)
2024-11-22T05:34:04,343 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741839_1015 (size=7739)
2024-11-22T05:34:06,321 INFO  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:34:08,267 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1efc7679337ac59f43f1b3f9a6d90462, had cached 0 bytes from a total of 25018
2024-11-22T05:34:08,525 INFO  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:34:10,729 INFO  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:34:12,934 INFO  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:34:14,937 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-22T05:34:14,938 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42405%2C1732253599913.1732253654938
2024-11-22T05:34:18,148 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-22T05:34:19,949 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5007 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:34:19,952 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5007 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:34:19,952 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 82891a793a29%2C42405%2C1732253599913:(num 1732253654938) roll requested
2024-11-22T05:34:19,952 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:19,952 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:19,952 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:19,953 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:19,953 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:19,953 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253644124 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253654938
2024-11-22T05:34:19,956 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741841_1017 (size=4753)
2024-11-22T05:34:19,957 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741841_1017 (size=4753)
2024-11-22T05:34:19,958 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38455:38455),(127.0.0.1/127.0.0.1:43135:43135)]
2024-11-22T05:34:19,958 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253644124 is not closed yet, will try archiving it next time
2024-11-22T05:34:19,958 INFO  [regionserver/82891a793a29:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42405%2C1732253599913.1732253659958
2024-11-22T05:34:24,962 INFO  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5002 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:34:24,963 WARN  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5002 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:34:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42405 {}] regionserver.HRegion(8855): Flush requested on 1efc7679337ac59f43f1b3f9a6d90462
2024-11-22T05:34:24,964 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1efc7679337ac59f43f1b3f9a6d90462 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T05:34:24,971 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5008 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:34:24,971 WARN  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5008 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:34:26,964 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-22T05:34:29,968 INFO  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5002 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:34:29,968 WARN  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5002 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:34:29,968 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:29,969 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:29,969 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:29,970 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:29,970 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:29,971 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253654938 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253659958
2024-11-22T05:34:29,972 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38455:38455),(127.0.0.1/127.0.0.1:43135:43135)]
2024-11-22T05:34:29,973 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253654938 is not closed yet, will try archiving it next time
2024-11-22T05:34:29,973 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 82891a793a29%2C42405%2C1732253599913:(num 1732253659958) roll requested
2024-11-22T05:34:29,974 INFO  [regionserver/82891a793a29:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42405%2C1732253599913.1732253669973
2024-11-22T05:34:29,974 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741842_1018 (size=1569)
2024-11-22T05:34:29,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741842_1018 (size=1569)
2024-11-22T05:34:29,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/094b2bd3baa04a5f98b6331dff65fbda is 1080, key is row0015/info:/1732253641703/Put/seqid=0
2024-11-22T05:34:29,990 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741844_1020 (size=12509)
2024-11-22T05:34:29,990 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741844_1020 (size=12509)
2024-11-22T05:34:29,991 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/094b2bd3baa04a5f98b6331dff65fbda
2024-11-22T05:34:30,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/094b2bd3baa04a5f98b6331dff65fbda as hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/094b2bd3baa04a5f98b6331dff65fbda
2024-11-22T05:34:30,013 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/094b2bd3baa04a5f98b6331dff65fbda, entries=7, sequenceid=31, filesize=12.2 K
2024-11-22T05:34:34,992 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5012 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:34:34,992 WARN  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5012 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:34:35,015 INFO  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:34:35,015 WARN  [FSHLog-0-hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b-prefix:82891a793a29,42405,1732253599913 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37373,DS-4a75f67d-84e4-4b55-8b3c-337c285ed7ca,DISK], DatanodeInfoWithStorage[127.0.0.1:41279,DS-85dcf605-e916-4551-87df-842e9e4a313c,DISK]]
2024-11-22T05:34:35,015 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:35,015 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 1efc7679337ac59f43f1b3f9a6d90462 in 10052ms, sequenceid=31, compaction requested=true
2024-11-22T05:34:35,016 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:35,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1efc7679337ac59f43f1b3f9a6d90462:

2024-11-22T05:34:35,016 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:35,016 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:35,016 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-11-22T05:34:35,016 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:35,016 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:34:35,016 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253659958 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253669973
2024-11-22T05:34:35,016 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/d2217d29dc7342daab521b589a2c8d40 because midkey is the same as first or last row
2024-11-22T05:34:35,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1efc7679337ac59f43f1b3f9a6d90462:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T05:34:35,018 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741843_1019 (size=438)
2024-11-22T05:34:35,019 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741843_1019 (size=438)
2024-11-22T05:34:35,019 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43135:43135),(127.0.0.1/127.0.0.1:38455:38455)]
2024-11-22T05:34:35,019 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253659958 is not closed yet, will try archiving it next time
2024-11-22T05:34:35,019 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253632654 to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/oldWALs/82891a793a29%2C42405%2C1732253599913.1732253632654
2024-11-22T05:34:35,019 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 82891a793a29%2C42405%2C1732253599913:(num 1732253669973) roll requested
2024-11-22T05:34:35,020 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42405%2C1732253599913.1732253675019
2024-11-22T05:34:35,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:34:35,021 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T05:34:35,021 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253644124 to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/oldWALs/82891a793a29%2C42405%2C1732253599913.1732253644124
2024-11-22T05:34:35,023 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253654938 to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/oldWALs/82891a793a29%2C42405%2C1732253599913.1732253654938
2024-11-22T05:34:35,024 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T05:34:35,024 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253659958 to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/oldWALs/82891a793a29%2C42405%2C1732253599913.1732253659958
2024-11-22T05:34:35,026 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.HStore(1541): 1efc7679337ac59f43f1b3f9a6d90462/info is initiating minor compaction (all files)
2024-11-22T05:34:35,027 INFO  [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1efc7679337ac59f43f1b3f9a6d90462/info in TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.
2024-11-22T05:34:35,027 INFO  [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/d2217d29dc7342daab521b589a2c8d40, hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/95524d1bb75a451185ad65fbc3d29bde, hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/094b2bd3baa04a5f98b6331dff65fbda] into tmpdir=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp, totalSize=36.6 K
2024-11-22T05:34:35,029 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2217d29dc7342daab521b589a2c8d40, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732253612604
2024-11-22T05:34:35,030 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95524d1bb75a451185ad65fbc3d29bde, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1732253626641
2024-11-22T05:34:35,031 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] compactions.Compactor(225): Compacting 094b2bd3baa04a5f98b6331dff65fbda, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732253641703
2024-11-22T05:34:35,034 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:35,034 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:35,034 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:35,035 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:35,035 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:35,035 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253669973 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253675019
2024-11-22T05:34:35,038 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741845_1021 (size=93)
2024-11-22T05:34:35,039 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741845_1021 (size=93)
2024-11-22T05:34:35,040 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253669973 to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/oldWALs/82891a793a29%2C42405%2C1732253599913.1732253669973
2024-11-22T05:34:35,041 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38455:38455),(127.0.0.1/127.0.0.1:43135:43135)]
2024-11-22T05:34:35,041 INFO  [regionserver/82891a793a29:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42405%2C1732253599913.1732253675041
2024-11-22T05:34:35,050 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:35,050 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:35,050 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:35,050 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:35,051 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:35,051 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253675019 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/WALs/82891a793a29,42405,1732253599913/82891a793a29%2C42405%2C1732253599913.1732253675041
2024-11-22T05:34:35,053 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741846_1022 (size=1258)
2024-11-22T05:34:35,053 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741846_1022 (size=1258)
2024-11-22T05:34:35,058 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38455:38455),(127.0.0.1/127.0.0.1:43135:43135)]
2024-11-22T05:34:35,068 INFO  [RS:0;82891a793a29:42405-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1efc7679337ac59f43f1b3f9a6d90462#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T05:34:35,069 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/68745de2d4ed4cdc92372f4bc918169c is 1080, key is row0001/info:/1732253612604/Put/seqid=0
2024-11-22T05:34:35,074 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741848_1024 (size=27710)
2024-11-22T05:34:35,075 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741848_1024 (size=27710)
2024-11-22T05:34:35,086 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/68745de2d4ed4cdc92372f4bc918169c as hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/68745de2d4ed4cdc92372f4bc918169c
2024-11-22T05:34:35,104 INFO  [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1efc7679337ac59f43f1b3f9a6d90462/info of 1efc7679337ac59f43f1b3f9a6d90462 into 68745de2d4ed4cdc92372f4bc918169c(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T05:34:35,104 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1efc7679337ac59f43f1b3f9a6d90462:

2024-11-22T05:34:35,106 INFO  [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462., storeName=1efc7679337ac59f43f1b3f9a6d90462/info, priority=13, startTime=1732253675018; duration=0sec
2024-11-22T05:34:35,106 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-22T05:34:35,106 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:34:35,107 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/68745de2d4ed4cdc92372f4bc918169c because midkey is the same as first or last row
2024-11-22T05:34:35,107 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-22T05:34:35,107 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:34:35,107 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/68745de2d4ed4cdc92372f4bc918169c because midkey is the same as first or last row
2024-11-22T05:34:35,107 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-22T05:34:35,107 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:34:35,107 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/68745de2d4ed4cdc92372f4bc918169c because midkey is the same as first or last row
2024-11-22T05:34:35,107 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:34:35,108 DEBUG [RS:0;82891a793a29:42405-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1efc7679337ac59f43f1b3f9a6d90462:info
2024-11-22T05:34:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42405 {}] regionserver.HRegion(8855): Flush requested on 1efc7679337ac59f43f1b3f9a6d90462
2024-11-22T05:34:47,077 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1efc7679337ac59f43f1b3f9a6d90462 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T05:34:47,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/0c83fefdc1ba453db7a067af5c1c7695 is 1080, key is row0022/info:/1732253675043/Put/seqid=0
2024-11-22T05:34:47,093 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741849_1025 (size=12509)
2024-11-22T05:34:47,094 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741849_1025 (size=12509)
2024-11-22T05:34:47,094 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/0c83fefdc1ba453db7a067af5c1c7695
2024-11-22T05:34:47,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/0c83fefdc1ba453db7a067af5c1c7695 as hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/0c83fefdc1ba453db7a067af5c1c7695
2024-11-22T05:34:47,113 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/0c83fefdc1ba453db7a067af5c1c7695, entries=7, sequenceid=42, filesize=12.2 K
2024-11-22T05:34:47,114 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 1efc7679337ac59f43f1b3f9a6d90462 in 37ms, sequenceid=42, compaction requested=false
2024-11-22T05:34:47,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1efc7679337ac59f43f1b3f9a6d90462:

2024-11-22T05:34:47,115 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-11-22T05:34:47,115 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:34:47,115 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/68745de2d4ed4cdc92372f4bc918169c because midkey is the same as first or last row
2024-11-22T05:34:48,149 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-22T05:34:53,268 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1efc7679337ac59f43f1b3f9a6d90462, had cached 0 bytes from a total of 40219
2024-11-22T05:34:55,091 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-22T05:34:55,092 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T05:34:55,092 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:34:55,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:34:55,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:34:55,098 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-22T05:34:55,098 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-22T05:34:55,098 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=887671035, stopped=false
2024-11-22T05:34:55,098 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=82891a793a29,43561,1732253599023
2024-11-22T05:34:55,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T05:34:55,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T05:34:55,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:34:55,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:34:55,151 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T05:34:55,151 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T05:34:55,152 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:34:55,152 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:34:55,152 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:34:55,152 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:34:55,152 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '82891a793a29,42405,1732253599913' *****
2024-11-22T05:34:55,152 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-22T05:34:55,153 INFO  [RS:0;82891a793a29:42405 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-22T05:34:55,153 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-22T05:34:55,153 INFO  [RS:0;82891a793a29:42405 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-22T05:34:55,153 INFO  [RS:0;82891a793a29:42405 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-22T05:34:55,153 INFO  [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(3091): Received CLOSE for 1efc7679337ac59f43f1b3f9a6d90462
2024-11-22T05:34:55,154 INFO  [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(959): stopping server 82891a793a29,42405,1732253599913
2024-11-22T05:34:55,154 INFO  [RS:0;82891a793a29:42405 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T05:34:55,154 INFO  [RS:0;82891a793a29:42405 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;82891a793a29:42405.
2024-11-22T05:34:55,154 DEBUG [RS:0;82891a793a29:42405 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:34:55,154 DEBUG [RS:0;82891a793a29:42405 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:34:55,154 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1efc7679337ac59f43f1b3f9a6d90462, disabling compactions & flushes
2024-11-22T05:34:55,154 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.
2024-11-22T05:34:55,154 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.
2024-11-22T05:34:55,154 INFO  [RS:0;82891a793a29:42405 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-22T05:34:55,154 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462. after waiting 0 ms
2024-11-22T05:34:55,154 INFO  [RS:0;82891a793a29:42405 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-22T05:34:55,154 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.
2024-11-22T05:34:55,154 INFO  [RS:0;82891a793a29:42405 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-22T05:34:55,154 INFO  [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-22T05:34:55,155 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1efc7679337ac59f43f1b3f9a6d90462 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-22T05:34:55,155 INFO  [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-22T05:34:55,155 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T05:34:55,155 DEBUG [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 1efc7679337ac59f43f1b3f9a6d90462=TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.}
2024-11-22T05:34:55,155 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T05:34:55,155 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T05:34:55,155 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T05:34:55,155 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T05:34:55,156 DEBUG [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1efc7679337ac59f43f1b3f9a6d90462
2024-11-22T05:34:55,156 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-11-22T05:34:55,160 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/58211fa88b244357a29931a77e694927 is 1080, key is row0029/info:/1732253689079/Put/seqid=0
2024-11-22T05:34:55,176 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741850_1026 (size=8193)
2024-11-22T05:34:55,176 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741850_1026 (size=8193)
2024-11-22T05:34:55,179 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/.tmp/info/d5f2a4d22ddc475380ba4c3d67048e76 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462./info:regioninfo/1732253603292/Put/seqid=0
2024-11-22T05:34:55,181 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/58211fa88b244357a29931a77e694927
2024-11-22T05:34:55,209 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741851_1027 (size=7016)
2024-11-22T05:34:55,210 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741851_1027 (size=7016)
2024-11-22T05:34:55,217 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/.tmp/info/58211fa88b244357a29931a77e694927 as hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/58211fa88b244357a29931a77e694927
2024-11-22T05:34:55,230 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/58211fa88b244357a29931a77e694927, entries=3, sequenceid=48, filesize=8.0 K
2024-11-22T05:34:55,232 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1efc7679337ac59f43f1b3f9a6d90462 in 77ms, sequenceid=48, compaction requested=true
2024-11-22T05:34:55,233 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/d2217d29dc7342daab521b589a2c8d40, hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/95524d1bb75a451185ad65fbc3d29bde, hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/094b2bd3baa04a5f98b6331dff65fbda] to archive
2024-11-22T05:34:55,237 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-22T05:34:55,242 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/d2217d29dc7342daab521b589a2c8d40 to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/archive/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/d2217d29dc7342daab521b589a2c8d40
2024-11-22T05:34:55,245 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/95524d1bb75a451185ad65fbc3d29bde to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/archive/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/95524d1bb75a451185ad65fbc3d29bde
2024-11-22T05:34:55,248 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/094b2bd3baa04a5f98b6331dff65fbda to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/archive/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/info/094b2bd3baa04a5f98b6331dff65fbda
2024-11-22T05:34:55,263 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=82891a793a29:43561 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-22T05:34:55,268 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [d2217d29dc7342daab521b589a2c8d40=12509, 95524d1bb75a451185ad65fbc3d29bde=12509, 094b2bd3baa04a5f98b6331dff65fbda=12509]
2024-11-22T05:34:55,275 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/default/TestLogRolling-testSlowSyncLogRolling/1efc7679337ac59f43f1b3f9a6d90462/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-11-22T05:34:55,278 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.
2024-11-22T05:34:55,278 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1efc7679337ac59f43f1b3f9a6d90462:
Waiting for close lock at 1732253695154Running coprocessor pre-close hooks at 1732253695154Disabling compacts and flushes for region at 1732253695154Disabling writes for close at 1732253695154Obtaining lock to block concurrent updates at 1732253695155 (+1 ms)Preparing flush snapshotting stores in 1efc7679337ac59f43f1b3f9a6d90462 at 1732253695155Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1732253695155Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462. at 1732253695156 (+1 ms)Flushing 1efc7679337ac59f43f1b3f9a6d90462/info: creating writer at 1732253695156Flushing 1efc7679337ac59f43f1b3f9a6d90462/info: appending metadata at 1732253695160 (+4 ms)Flushing 1efc7679337ac59f43f1b3f9a6d90462/info: closing flushed file at 1732253695160Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54df96f3: reopening flushed file at 1732253695215 (+55 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1efc7679337ac59f43f1b3f9a6d90462 in 77ms, sequenceid=48, compaction requested=true at 1732253695232 (+17 ms)Writing region close event to WAL at 1732253695269 (+37 ms)Running coprocessor post-close hooks at 1732253695276 (+7 ms)Closed at 1732253695278 (+2 ms)
2024-11-22T05:34:55,279 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1732253602425.1efc7679337ac59f43f1b3f9a6d90462.
2024-11-22T05:34:55,347 INFO  [regionserver/82891a793a29:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T05:34:55,356 DEBUG [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-22T05:34:55,422 INFO  [regionserver/82891a793a29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-22T05:34:55,422 INFO  [regionserver/82891a793a29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-22T05:34:55,556 DEBUG [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-22T05:34:55,610 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/.tmp/info/d5f2a4d22ddc475380ba4c3d67048e76
2024-11-22T05:34:55,643 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/.tmp/ns/52e91e9980a84da7b827e99d33dfdb94 is 43, key is default/ns:d/1732253602209/Put/seqid=0
2024-11-22T05:34:55,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741852_1028 (size=5153)
2024-11-22T05:34:55,650 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741852_1028 (size=5153)
2024-11-22T05:34:55,650 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/.tmp/ns/52e91e9980a84da7b827e99d33dfdb94
2024-11-22T05:34:55,677 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/.tmp/table/eebb9bb454834f569d8805378e07db59 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1732253603307/Put/seqid=0
2024-11-22T05:34:55,684 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741853_1029 (size=5396)
2024-11-22T05:34:55,684 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741853_1029 (size=5396)
2024-11-22T05:34:55,685 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/.tmp/table/eebb9bb454834f569d8805378e07db59
2024-11-22T05:34:55,696 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/.tmp/info/d5f2a4d22ddc475380ba4c3d67048e76 as hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/info/d5f2a4d22ddc475380ba4c3d67048e76
2024-11-22T05:34:55,706 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/info/d5f2a4d22ddc475380ba4c3d67048e76, entries=10, sequenceid=11, filesize=6.9 K
2024-11-22T05:34:55,708 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/.tmp/ns/52e91e9980a84da7b827e99d33dfdb94 as hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/ns/52e91e9980a84da7b827e99d33dfdb94
2024-11-22T05:34:55,720 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/ns/52e91e9980a84da7b827e99d33dfdb94, entries=2, sequenceid=11, filesize=5.0 K
2024-11-22T05:34:55,722 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/.tmp/table/eebb9bb454834f569d8805378e07db59 as hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/table/eebb9bb454834f569d8805378e07db59
2024-11-22T05:34:55,735 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/table/eebb9bb454834f569d8805378e07db59, entries=2, sequenceid=11, filesize=5.3 K
2024-11-22T05:34:55,737 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 581ms, sequenceid=11, compaction requested=false
2024-11-22T05:34:55,745 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-22T05:34:55,746 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T05:34:55,746 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T05:34:55,747 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732253695155Running coprocessor pre-close hooks at 1732253695155Disabling compacts and flushes for region at 1732253695155Disabling writes for close at 1732253695155Obtaining lock to block concurrent updates at 1732253695156 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732253695156Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1732253695156Flushing stores of hbase:meta,,1.1588230740 at 1732253695157 (+1 ms)Flushing 1588230740/info: creating writer at 1732253695157Flushing 1588230740/info: appending metadata at 1732253695179 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732253695179Flushing 1588230740/ns: creating writer at 1732253695620 (+441 ms)Flushing 1588230740/ns: appending metadata at 1732253695642 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1732253695642Flushing 1588230740/table: creating writer at 1732253695659 (+17 ms)Flushing 1588230740/table: appending metadata at 1732253695676 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732253695677 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@538a1201: reopening flushed file at 1732253695694 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e016236: reopening flushed file at 1732253695706 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6eeb3a5b: reopening flushed file at 1732253695720 (+14 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 581ms, sequenceid=11, compaction requested=false at 1732253695737 (+17 ms)Writing region close event to WAL at 1732253695738 (+1 ms)Running coprocessor post-close hooks at 1732253695746 (+8 ms)Closed at 1732253695746
2024-11-22T05:34:55,747 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-22T05:34:55,757 INFO  [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(976): stopping server 82891a793a29,42405,1732253599913; all regions closed.
2024-11-22T05:34:55,758 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:55,758 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:55,759 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:55,759 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:55,759 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:55,762 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741834_1010 (size=3066)
2024-11-22T05:34:55,762 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741834_1010 (size=3066)
2024-11-22T05:34:55,767 DEBUG [RS:0;82891a793a29:42405 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/oldWALs
2024-11-22T05:34:55,767 INFO  [RS:0;82891a793a29:42405 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 82891a793a29%2C42405%2C1732253599913.meta:.meta(num 1732253602025)
2024-11-22T05:34:55,767 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:55,768 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:55,768 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:55,768 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:55,768 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:55,770 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741847_1023 (size=12695)
2024-11-22T05:34:55,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741847_1023 (size=12695)
2024-11-22T05:34:55,776 DEBUG [RS:0;82891a793a29:42405 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/oldWALs
2024-11-22T05:34:55,776 INFO  [RS:0;82891a793a29:42405 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 82891a793a29%2C42405%2C1732253599913:(num 1732253675041)
2024-11-22T05:34:55,776 DEBUG [RS:0;82891a793a29:42405 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:34:55,776 INFO  [RS:0;82891a793a29:42405 {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T05:34:55,776 INFO  [RS:0;82891a793a29:42405 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T05:34:55,776 INFO  [RS:0;82891a793a29:42405 {}] hbase.ChoreService(370): Chore service for: regionserver/82891a793a29:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-22T05:34:55,776 INFO  [RS:0;82891a793a29:42405 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T05:34:55,776 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T05:34:55,777 INFO  [RS:0;82891a793a29:42405 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42405
2024-11-22T05:34:55,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T05:34:55,824 INFO  [RS:0;82891a793a29:42405 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T05:34:55,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/82891a793a29,42405,1732253599913
2024-11-22T05:34:55,836 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [82891a793a29,42405,1732253599913]
2024-11-22T05:34:55,845 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/82891a793a29,42405,1732253599913 already deleted, retry=false
2024-11-22T05:34:55,846 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 82891a793a29,42405,1732253599913 expired; onlineServers=0
2024-11-22T05:34:55,846 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '82891a793a29,43561,1732253599023' *****
2024-11-22T05:34:55,846 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-22T05:34:55,846 INFO  [M:0;82891a793a29:43561 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T05:34:55,846 INFO  [M:0;82891a793a29:43561 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T05:34:55,846 DEBUG [M:0;82891a793a29:43561 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-22T05:34:55,847 DEBUG [M:0;82891a793a29:43561 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-22T05:34:55,847 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-22T05:34:55,847 DEBUG [master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253601164 {}] cleaner.HFileCleaner(306): Exit Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253601164,5,FailOnTimeoutGroup]
2024-11-22T05:34:55,847 DEBUG [master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253601158 {}] cleaner.HFileCleaner(306): Exit Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253601158,5,FailOnTimeoutGroup]
2024-11-22T05:34:55,847 INFO  [M:0;82891a793a29:43561 {}] hbase.ChoreService(370): Chore service for: master/82891a793a29:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-22T05:34:55,847 INFO  [M:0;82891a793a29:43561 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T05:34:55,847 DEBUG [M:0;82891a793a29:43561 {}] master.HMaster(1795): Stopping service threads
2024-11-22T05:34:55,847 INFO  [M:0;82891a793a29:43561 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-22T05:34:55,847 INFO  [M:0;82891a793a29:43561 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T05:34:55,848 INFO  [M:0;82891a793a29:43561 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-22T05:34:55,848 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-22T05:34:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-22T05:34:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:34:55,856 DEBUG [M:0;82891a793a29:43561 {}] zookeeper.ZKUtil(347): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-22T05:34:55,856 WARN  [M:0;82891a793a29:43561 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-22T05:34:55,857 INFO  [M:0;82891a793a29:43561 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/.lastflushedseqids
2024-11-22T05:34:55,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741854_1030 (size=130)
2024-11-22T05:34:55,873 INFO  [M:0;82891a793a29:43561 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-22T05:34:55,874 INFO  [M:0;82891a793a29:43561 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-22T05:34:55,874 DEBUG [M:0;82891a793a29:43561 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T05:34:55,874 INFO  [M:0;82891a793a29:43561 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:34:55,874 DEBUG [M:0;82891a793a29:43561 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:34:55,874 DEBUG [M:0;82891a793a29:43561 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T05:34:55,874 DEBUG [M:0;82891a793a29:43561 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:34:55,874 INFO  [M:0;82891a793a29:43561 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.01 KB heapSize=29.18 KB
2024-11-22T05:34:55,877 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741854_1030 (size=130)
2024-11-22T05:34:55,898 DEBUG [M:0;82891a793a29:43561 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/11ecb00b4e9449f09de2f76cd6cb49ee is 82, key is hbase:meta,,1/info:regioninfo/1732253602100/Put/seqid=0
2024-11-22T05:34:55,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741855_1031 (size=5672)
2024-11-22T05:34:55,911 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741855_1031 (size=5672)
2024-11-22T05:34:55,912 INFO  [M:0;82891a793a29:43561 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/11ecb00b4e9449f09de2f76cd6cb49ee
2024-11-22T05:34:55,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:34:55,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42405-0x10161083da50001, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:34:55,936 INFO  [RS:0;82891a793a29:42405 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T05:34:55,936 INFO  [RS:0;82891a793a29:42405 {}] regionserver.HRegionServer(1031): Exiting; stopping=82891a793a29,42405,1732253599913; zookeeper connection closed.
2024-11-22T05:34:55,937 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@50dd1a3b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@50dd1a3b
2024-11-22T05:34:55,937 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-22T05:34:55,941 DEBUG [M:0;82891a793a29:43561 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/77c9e5828a4d46959d9a1225824530c7 is 765, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732253603316/Put/seqid=0
2024-11-22T05:34:55,948 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741856_1032 (size=6246)
2024-11-22T05:34:55,949 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741856_1032 (size=6246)
2024-11-22T05:34:55,950 INFO  [M:0;82891a793a29:43561 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.41 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/77c9e5828a4d46959d9a1225824530c7
2024-11-22T05:34:55,958 INFO  [M:0;82891a793a29:43561 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 77c9e5828a4d46959d9a1225824530c7
2024-11-22T05:34:55,976 DEBUG [M:0;82891a793a29:43561 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d82eb23cdcd84e3a87e3bfbfb152ad18 is 69, key is 82891a793a29,42405,1732253599913/rs:state/1732253601258/Put/seqid=0
2024-11-22T05:34:55,986 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741857_1033 (size=5156)
2024-11-22T05:34:55,986 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741857_1033 (size=5156)
2024-11-22T05:34:55,987 INFO  [M:0;82891a793a29:43561 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d82eb23cdcd84e3a87e3bfbfb152ad18
2024-11-22T05:34:56,018 DEBUG [M:0;82891a793a29:43561 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b19ca741100f41abb2873e9b1b7b2544 is 52, key is load_balancer_on/state:d/1732253602385/Put/seqid=0
2024-11-22T05:34:56,024 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741858_1034 (size=5056)
2024-11-22T05:34:56,024 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741858_1034 (size=5056)
2024-11-22T05:34:56,025 INFO  [M:0;82891a793a29:43561 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b19ca741100f41abb2873e9b1b7b2544
2024-11-22T05:34:56,034 DEBUG [M:0;82891a793a29:43561 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/11ecb00b4e9449f09de2f76cd6cb49ee as hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/11ecb00b4e9449f09de2f76cd6cb49ee
2024-11-22T05:34:56,042 INFO  [M:0;82891a793a29:43561 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/11ecb00b4e9449f09de2f76cd6cb49ee, entries=8, sequenceid=59, filesize=5.5 K
2024-11-22T05:34:56,043 DEBUG [M:0;82891a793a29:43561 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/77c9e5828a4d46959d9a1225824530c7 as hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/77c9e5828a4d46959d9a1225824530c7
2024-11-22T05:34:56,052 INFO  [M:0;82891a793a29:43561 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 77c9e5828a4d46959d9a1225824530c7
2024-11-22T05:34:56,052 INFO  [M:0;82891a793a29:43561 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/77c9e5828a4d46959d9a1225824530c7, entries=6, sequenceid=59, filesize=6.1 K
2024-11-22T05:34:56,054 DEBUG [M:0;82891a793a29:43561 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d82eb23cdcd84e3a87e3bfbfb152ad18 as hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d82eb23cdcd84e3a87e3bfbfb152ad18
2024-11-22T05:34:56,062 INFO  [M:0;82891a793a29:43561 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d82eb23cdcd84e3a87e3bfbfb152ad18, entries=1, sequenceid=59, filesize=5.0 K
2024-11-22T05:34:56,064 DEBUG [M:0;82891a793a29:43561 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b19ca741100f41abb2873e9b1b7b2544 as hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b19ca741100f41abb2873e9b1b7b2544
2024-11-22T05:34:56,072 INFO  [M:0;82891a793a29:43561 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b19ca741100f41abb2873e9b1b7b2544, entries=1, sequenceid=59, filesize=4.9 K
2024-11-22T05:34:56,074 INFO  [M:0;82891a793a29:43561 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.01 KB/23564, heapSize ~29.12 KB/29816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 199ms, sequenceid=59, compaction requested=false
2024-11-22T05:34:56,075 INFO  [M:0;82891a793a29:43561 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:34:56,075 DEBUG [M:0;82891a793a29:43561 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732253695874Disabling compacts and flushes for region at 1732253695874Disabling writes for close at 1732253695874Obtaining lock to block concurrent updates at 1732253695874Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732253695874Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23564, getHeapSize=29816, getOffHeapSize=0, getCellsCount=70 at 1732253695875 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732253695877 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732253695877Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732253695897 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732253695897Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732253695921 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732253695940 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732253695940Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732253695958 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732253695975 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732253695975Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732253695999 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732253696017 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732253696017Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d7173a6: reopening flushed file at 1732253696033 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19f5de74: reopening flushed file at 1732253696042 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17922cf4: reopening flushed file at 1732253696052 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7825dc7f: reopening flushed file at 1732253696063 (+11 ms)Finished flush of dataSize ~23.01 KB/23564, heapSize ~29.12 KB/29816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 199ms, sequenceid=59, compaction requested=false at 1732253696074 (+11 ms)Writing region close event to WAL at 1732253696075 (+1 ms)Closed at 1732253696075
2024-11-22T05:34:56,077 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:56,077 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:56,077 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:56,077 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:56,077 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:34:56,079 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41279 is added to blk_1073741830_1006 (size=27961)
2024-11-22T05:34:56,080 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37373 is added to blk_1073741830_1006 (size=27961)
2024-11-22T05:34:56,081 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T05:34:56,081 INFO  [M:0;82891a793a29:43561 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-22T05:34:56,081 INFO  [M:0;82891a793a29:43561 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43561
2024-11-22T05:34:56,081 INFO  [M:0;82891a793a29:43561 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T05:34:56,188 INFO  [M:0;82891a793a29:43561 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T05:34:56,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:34:56,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43561-0x10161083da50000, quorum=127.0.0.1:54019, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:34:56,206 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:34:56,209 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:34:56,210 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:34:56,210 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:34:56,210 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/hadoop.log.dir/,STOPPED}
2024-11-22T05:34:56,214 WARN  [BP-1033121826-172.17.0.2-1732253595128 heartbeating to localhost/127.0.0.1:40545 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:34:56,214 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:34:56,214 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:34:56,214 WARN  [BP-1033121826-172.17.0.2-1732253595128 heartbeating to localhost/127.0.0.1:40545 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1033121826-172.17.0.2-1732253595128 (Datanode Uuid 7f3bcfa2-3469-4bda-8d6c-03316cb48ec7) service to localhost/127.0.0.1:40545
2024-11-22T05:34:56,216 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/cluster_986ee2ba-9b27-0ddc-1f6c-d6f5e45fb4ac/data/data3/current/BP-1033121826-172.17.0.2-1732253595128 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:34:56,216 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/cluster_986ee2ba-9b27-0ddc-1f6c-d6f5e45fb4ac/data/data4/current/BP-1033121826-172.17.0.2-1732253595128 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:34:56,216 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:34:56,218 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:34:56,219 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:34:56,219 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:34:56,219 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:34:56,219 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/hadoop.log.dir/,STOPPED}
2024-11-22T05:34:56,221 WARN  [BP-1033121826-172.17.0.2-1732253595128 heartbeating to localhost/127.0.0.1:40545 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:34:56,221 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:34:56,222 WARN  [BP-1033121826-172.17.0.2-1732253595128 heartbeating to localhost/127.0.0.1:40545 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1033121826-172.17.0.2-1732253595128 (Datanode Uuid 090d0970-9579-4fee-bb82-b7d4a1142ea7) service to localhost/127.0.0.1:40545
2024-11-22T05:34:56,222 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:34:56,222 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/cluster_986ee2ba-9b27-0ddc-1f6c-d6f5e45fb4ac/data/data1/current/BP-1033121826-172.17.0.2-1732253595128 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:34:56,222 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/cluster_986ee2ba-9b27-0ddc-1f6c-d6f5e45fb4ac/data/data2/current/BP-1033121826-172.17.0.2-1732253595128 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:34:56,223 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:34:56,233 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T05:34:56,233 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:34:56,234 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:34:56,234 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:34:56,234 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/hadoop.log.dir/,STOPPED}
2024-11-22T05:34:56,247 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-22T05:34:56,295 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-22T05:34:56,307 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=78 (was 12)
Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/82891a793a29:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40545 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40545 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40545 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: master/82891a793a29:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:40545
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40545
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40545
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:40545
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40545
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: master/82891a793a29:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@6f391bc8
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=402 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=189 (was 196), ProcessCount=11 (was 11), AvailableMemoryMB=8065 (was 8556)
2024-11-22T05:34:56,315 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=79, OpenFileDescriptor=402, MaxFileDescriptor=1048576, SystemLoadAverage=189, ProcessCount=11, AvailableMemoryMB=8063
2024-11-22T05:34:56,315 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-22T05:34:56,316 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/hadoop.log.dir so I do NOT create it in target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b
2024-11-22T05:34:56,316 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e2ab7ec9-3969-4b47-864d-f14f287e4944/hadoop.tmp.dir so I do NOT create it in target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b
2024-11-22T05:34:56,316 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/cluster_b7ed9aed-9d3c-9602-ce42-fac914215dcb, deleteOnExit=true
2024-11-22T05:34:56,316 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-22T05:34:56,316 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/test.cache.data in system properties and HBase conf
2024-11-22T05:34:56,317 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/hadoop.tmp.dir in system properties and HBase conf
2024-11-22T05:34:56,317 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/hadoop.log.dir in system properties and HBase conf
2024-11-22T05:34:56,317 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-22T05:34:56,317 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-22T05:34:56,317 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-22T05:34:56,317 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-22T05:34:56,317 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-22T05:34:56,317 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-22T05:34:56,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-22T05:34:56,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T05:34:56,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-22T05:34:56,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-22T05:34:56,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T05:34:56,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T05:34:56,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-22T05:34:56,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/nfs.dump.dir in system properties and HBase conf
2024-11-22T05:34:56,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/java.io.tmpdir in system properties and HBase conf
2024-11-22T05:34:56,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T05:34:56,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-22T05:34:56,319 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-22T05:34:56,339 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T05:34:56,763 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:34:56,771 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:34:56,776 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:34:56,776 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:34:56,776 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:34:56,777 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:34:56,778 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@737d6115{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:34:56,778 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@641eaf99{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:34:56,893 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2a7d11e5{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/java.io.tmpdir/jetty-localhost-41645-hadoop-hdfs-3_4_1-tests_jar-_-any-2446187597134647649/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T05:34:56,894 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5d4bdc00{HTTP/1.1, (http/1.1)}{localhost:41645}
2024-11-22T05:34:56,894 INFO  [Time-limited test {}] server.Server(415): Started @104152ms
2024-11-22T05:34:56,909 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T05:34:57,214 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:34:57,220 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:34:57,226 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:34:57,226 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:34:57,226 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:34:57,232 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b58749b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:34:57,233 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1aa9c156{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:34:57,371 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@66b9e907{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/java.io.tmpdir/jetty-localhost-40213-hadoop-hdfs-3_4_1-tests_jar-_-any-14413251765241952706/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:34:57,371 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@51f8cb4f{HTTP/1.1, (http/1.1)}{localhost:40213}
2024-11-22T05:34:57,371 INFO  [Time-limited test {}] server.Server(415): Started @104629ms
2024-11-22T05:34:57,373 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:34:57,428 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:34:57,435 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:34:57,436 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:34:57,436 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:34:57,436 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:34:57,437 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a742c1f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:34:57,438 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d5e070a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:34:57,584 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c17f9e5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/java.io.tmpdir/jetty-localhost-41555-hadoop-hdfs-3_4_1-tests_jar-_-any-16233318477610869873/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:34:57,585 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@669fbf01{HTTP/1.1, (http/1.1)}{localhost:41555}
2024-11-22T05:34:57,585 INFO  [Time-limited test {}] server.Server(415): Started @104843ms
2024-11-22T05:34:57,588 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:34:58,445 WARN  [Thread-451 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/cluster_b7ed9aed-9d3c-9602-ce42-fac914215dcb/data/data1/current/BP-1370325539-172.17.0.2-1732253696356/current, will proceed with Du for space computation calculation, 
2024-11-22T05:34:58,449 WARN  [Thread-452 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/cluster_b7ed9aed-9d3c-9602-ce42-fac914215dcb/data/data2/current/BP-1370325539-172.17.0.2-1732253696356/current, will proceed with Du for space computation calculation, 
2024-11-22T05:34:58,477 WARN  [Thread-416 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:34:58,480 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x27a791cf6fc36eb1 with lease ID 0x3956072e1ce5b7b6: Processing first storage report for DS-d19dcba8-ad90-499c-8aeb-44ae79fad056 from datanode DatanodeRegistration(127.0.0.1:46801, datanodeUuid=285f8640-8b38-4fc0-ae10-71cd72d55cf8, infoPort=42897, infoSecurePort=0, ipcPort=36461, storageInfo=lv=-57;cid=testClusterID;nsid=25674226;c=1732253696356)
2024-11-22T05:34:58,481 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x27a791cf6fc36eb1 with lease ID 0x3956072e1ce5b7b6: from storage DS-d19dcba8-ad90-499c-8aeb-44ae79fad056 node DatanodeRegistration(127.0.0.1:46801, datanodeUuid=285f8640-8b38-4fc0-ae10-71cd72d55cf8, infoPort=42897, infoSecurePort=0, ipcPort=36461, storageInfo=lv=-57;cid=testClusterID;nsid=25674226;c=1732253696356), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:34:58,481 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x27a791cf6fc36eb1 with lease ID 0x3956072e1ce5b7b6: Processing first storage report for DS-e947b394-31b1-4fce-8d67-23d0a40516e5 from datanode DatanodeRegistration(127.0.0.1:46801, datanodeUuid=285f8640-8b38-4fc0-ae10-71cd72d55cf8, infoPort=42897, infoSecurePort=0, ipcPort=36461, storageInfo=lv=-57;cid=testClusterID;nsid=25674226;c=1732253696356)
2024-11-22T05:34:58,481 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x27a791cf6fc36eb1 with lease ID 0x3956072e1ce5b7b6: from storage DS-e947b394-31b1-4fce-8d67-23d0a40516e5 node DatanodeRegistration(127.0.0.1:46801, datanodeUuid=285f8640-8b38-4fc0-ae10-71cd72d55cf8, infoPort=42897, infoSecurePort=0, ipcPort=36461, storageInfo=lv=-57;cid=testClusterID;nsid=25674226;c=1732253696356), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:34:58,666 WARN  [Thread-463 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/cluster_b7ed9aed-9d3c-9602-ce42-fac914215dcb/data/data3/current/BP-1370325539-172.17.0.2-1732253696356/current, will proceed with Du for space computation calculation, 
2024-11-22T05:34:58,666 WARN  [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/cluster_b7ed9aed-9d3c-9602-ce42-fac914215dcb/data/data4/current/BP-1370325539-172.17.0.2-1732253696356/current, will proceed with Du for space computation calculation, 
2024-11-22T05:34:58,689 WARN  [Thread-439 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:34:58,692 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5075200b8c4139d5 with lease ID 0x3956072e1ce5b7b7: Processing first storage report for DS-a0298bb6-bf82-490b-8e9c-b1377d9e1fc9 from datanode DatanodeRegistration(127.0.0.1:35947, datanodeUuid=8ead25df-fb9e-4f76-9e9f-fe7257afe5ac, infoPort=42275, infoSecurePort=0, ipcPort=38111, storageInfo=lv=-57;cid=testClusterID;nsid=25674226;c=1732253696356)
2024-11-22T05:34:58,692 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5075200b8c4139d5 with lease ID 0x3956072e1ce5b7b7: from storage DS-a0298bb6-bf82-490b-8e9c-b1377d9e1fc9 node DatanodeRegistration(127.0.0.1:35947, datanodeUuid=8ead25df-fb9e-4f76-9e9f-fe7257afe5ac, infoPort=42275, infoSecurePort=0, ipcPort=38111, storageInfo=lv=-57;cid=testClusterID;nsid=25674226;c=1732253696356), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:34:58,692 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5075200b8c4139d5 with lease ID 0x3956072e1ce5b7b7: Processing first storage report for DS-20412e72-6093-47df-ba51-91c7f7501c38 from datanode DatanodeRegistration(127.0.0.1:35947, datanodeUuid=8ead25df-fb9e-4f76-9e9f-fe7257afe5ac, infoPort=42275, infoSecurePort=0, ipcPort=38111, storageInfo=lv=-57;cid=testClusterID;nsid=25674226;c=1732253696356)
2024-11-22T05:34:58,692 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5075200b8c4139d5 with lease ID 0x3956072e1ce5b7b7: from storage DS-20412e72-6093-47df-ba51-91c7f7501c38 node DatanodeRegistration(127.0.0.1:35947, datanodeUuid=8ead25df-fb9e-4f76-9e9f-fe7257afe5ac, infoPort=42275, infoSecurePort=0, ipcPort=38111, storageInfo=lv=-57;cid=testClusterID;nsid=25674226;c=1732253696356), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:34:58,733 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b
2024-11-22T05:34:58,736 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/cluster_b7ed9aed-9d3c-9602-ce42-fac914215dcb/zookeeper_0, clientPort=59969, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/cluster_b7ed9aed-9d3c-9602-ce42-fac914215dcb/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/cluster_b7ed9aed-9d3c-9602-ce42-fac914215dcb/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-22T05:34:58,737 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59969
2024-11-22T05:34:58,738 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:34:58,740 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:34:58,753 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46801 is added to blk_1073741825_1001 (size=7)
2024-11-22T05:34:58,754 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35947 is added to blk_1073741825_1001 (size=7)
2024-11-22T05:34:58,755 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363 with version=8
2024-11-22T05:34:58,755 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/hbase-staging
2024-11-22T05:34:58,758 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/82891a793a29:0 server-side Connection retries=45
2024-11-22T05:34:58,759 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:34:58,759 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T05:34:58,759 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T05:34:58,759 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:34:58,759 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T05:34:58,759 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-22T05:34:58,759 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T05:34:58,760 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39963
2024-11-22T05:34:58,763 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39963 connecting to ZooKeeper ensemble=127.0.0.1:59969
2024-11-22T05:34:58,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:399630x0, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T05:34:58,841 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39963-0x1016109c65b0000 connected
2024-11-22T05:34:58,922 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:34:58,925 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:34:58,928 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:34:58,928 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363, hbase.cluster.distributed=false
2024-11-22T05:34:58,931 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T05:34:58,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39963
2024-11-22T05:34:58,937 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39963
2024-11-22T05:34:58,937 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39963
2024-11-22T05:34:58,938 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39963
2024-11-22T05:34:58,938 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39963
2024-11-22T05:34:58,959 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/82891a793a29:0 server-side Connection retries=45
2024-11-22T05:34:58,959 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:34:58,959 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T05:34:58,959 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T05:34:58,959 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:34:58,959 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T05:34:58,960 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-22T05:34:58,960 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T05:34:58,964 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32879
2024-11-22T05:34:58,967 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32879 connecting to ZooKeeper ensemble=127.0.0.1:59969
2024-11-22T05:34:58,968 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:34:58,973 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:34:59,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:328790x0, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T05:34:59,006 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:328790x0, quorum=127.0.0.1:59969, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:34:59,006 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32879-0x1016109c65b0001 connected
2024-11-22T05:34:59,007 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-22T05:34:59,007 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-22T05:34:59,008 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-22T05:34:59,011 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T05:34:59,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32879
2024-11-22T05:34:59,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32879
2024-11-22T05:34:59,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32879
2024-11-22T05:34:59,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32879
2024-11-22T05:34:59,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32879
2024-11-22T05:34:59,031 DEBUG [M:0;82891a793a29:39963 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;82891a793a29:39963
2024-11-22T05:34:59,033 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/82891a793a29,39963,1732253698758
2024-11-22T05:34:59,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:34:59,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:34:59,038 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/82891a793a29,39963,1732253698758
2024-11-22T05:34:59,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:34:59,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-22T05:34:59,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:34:59,048 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-22T05:34:59,049 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/82891a793a29,39963,1732253698758 from backup master directory
2024-11-22T05:34:59,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/82891a793a29,39963,1732253698758
2024-11-22T05:34:59,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:34:59,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:34:59,063 WARN  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T05:34:59,063 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=82891a793a29,39963,1732253698758
2024-11-22T05:34:59,076 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/hbase.id] with ID: 9a354998-d724-47a4-9962-bec68dd1d92f
2024-11-22T05:34:59,076 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/.tmp/hbase.id
2024-11-22T05:34:59,110 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35947 is added to blk_1073741826_1002 (size=42)
2024-11-22T05:34:59,110 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46801 is added to blk_1073741826_1002 (size=42)
2024-11-22T05:34:59,113 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/.tmp/hbase.id]:[hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/hbase.id]
2024-11-22T05:34:59,136 INFO  [master/82891a793a29:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:34:59,136 INFO  [master/82891a793a29:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-22T05:34:59,139 INFO  [master/82891a793a29:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms.
2024-11-22T05:34:59,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:34:59,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:34:59,186 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35947 is added to blk_1073741827_1003 (size=196)
2024-11-22T05:34:59,187 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46801 is added to blk_1073741827_1003 (size=196)
2024-11-22T05:34:59,188 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T05:34:59,189 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-22T05:34:59,192 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:34:59,218 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46801 is added to blk_1073741828_1004 (size=1189)
2024-11-22T05:34:59,219 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35947 is added to blk_1073741828_1004 (size=1189)
2024-11-22T05:34:59,220 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store
2024-11-22T05:34:59,230 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46801 is added to blk_1073741829_1005 (size=34)
2024-11-22T05:34:59,230 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35947 is added to blk_1073741829_1005 (size=34)
2024-11-22T05:34:59,231 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:34:59,232 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T05:34:59,232 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:34:59,232 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:34:59,232 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T05:34:59,232 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:34:59,232 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:34:59,232 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732253699232Disabling compacts and flushes for region at 1732253699232Disabling writes for close at 1732253699232Writing region close event to WAL at 1732253699232Closed at 1732253699232
2024-11-22T05:34:59,234 WARN  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/.initializing
2024-11-22T05:34:59,234 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/WALs/82891a793a29,39963,1732253698758
2024-11-22T05:34:59,238 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C39963%2C1732253698758, suffix=, logDir=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/WALs/82891a793a29,39963,1732253698758, archiveDir=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/oldWALs, maxLogs=10
2024-11-22T05:34:59,239 INFO  [master/82891a793a29:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C39963%2C1732253698758.1732253699238
2024-11-22T05:34:59,251 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/WALs/82891a793a29,39963,1732253698758/82891a793a29%2C39963%2C1732253698758.1732253699238
2024-11-22T05:34:59,253 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42897:42897),(127.0.0.1/127.0.0.1:42275:42275)]
2024-11-22T05:34:59,253 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:34:59,254 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:34:59,254 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:34:59,254 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:34:59,256 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:34:59,259 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-22T05:34:59,259 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:34:59,260 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:34:59,260 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:34:59,262 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-22T05:34:59,262 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:34:59,263 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:34:59,263 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:34:59,266 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-22T05:34:59,266 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:34:59,267 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:34:59,267 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:34:59,269 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-22T05:34:59,270 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:34:59,271 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:34:59,271 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:34:59,272 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:34:59,273 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:34:59,275 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:34:59,275 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:34:59,276 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-22T05:34:59,278 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:34:59,289 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:34:59,290 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=765501, jitterRate=-0.026615485548973083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-22T05:34:59,292 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732253699254Initializing all the Stores at 1732253699256 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253699256Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253699256Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253699256Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253699256Cleaning up temporary data from old regions at 1732253699275 (+19 ms)Region opened successfully at 1732253699292 (+17 ms)
2024-11-22T05:34:59,292 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-22T05:34:59,298 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aaed38e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82891a793a29/172.17.0.2:0
2024-11-22T05:34:59,299 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-22T05:34:59,300 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-22T05:34:59,300 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-22T05:34:59,300 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-22T05:34:59,301 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-22T05:34:59,302 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-22T05:34:59,302 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-22T05:34:59,306 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-22T05:34:59,307 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-22T05:34:59,311 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-22T05:34:59,311 INFO  [master/82891a793a29:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-22T05:34:59,313 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-22T05:34:59,321 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-22T05:34:59,322 INFO  [master/82891a793a29:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-22T05:34:59,323 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-22T05:34:59,332 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-22T05:34:59,334 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-22T05:34:59,342 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-22T05:34:59,346 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-22T05:34:59,353 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-22T05:34:59,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T05:34:59,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T05:34:59,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:34:59,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:34:59,366 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=82891a793a29,39963,1732253698758, sessionid=0x1016109c65b0000, setting cluster-up flag (Was=false)
2024-11-22T05:34:59,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:34:59,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:34:59,416 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-22T05:34:59,418 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=82891a793a29,39963,1732253698758
2024-11-22T05:34:59,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:34:59,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:34:59,469 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-22T05:34:59,470 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=82891a793a29,39963,1732253698758
2024-11-22T05:34:59,471 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-22T05:34:59,473 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-22T05:34:59,474 INFO  [master/82891a793a29:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-22T05:34:59,474 INFO  [master/82891a793a29:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-22T05:34:59,474 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 82891a793a29,39963,1732253698758
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-22T05:34:59,476 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:34:59,476 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:34:59,476 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:34:59,476 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:34:59,476 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/82891a793a29:0, corePoolSize=10, maxPoolSize=10
2024-11-22T05:34:59,476 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:34:59,476 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/82891a793a29:0, corePoolSize=2, maxPoolSize=2
2024-11-22T05:34:59,477 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:34:59,478 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732253729477
2024-11-22T05:34:59,478 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-22T05:34:59,478 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-22T05:34:59,478 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-22T05:34:59,478 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-22T05:34:59,478 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-22T05:34:59,478 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-22T05:34:59,479 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T05:34:59,479 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:34:59,479 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-22T05:34:59,480 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-22T05:34:59,480 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-22T05:34:59,480 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-22T05:34:59,480 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-22T05:34:59,481 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-22T05:34:59,481 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:34:59,481 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253699481,5,FailOnTimeoutGroup]
2024-11-22T05:34:59,481 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253699481,5,FailOnTimeoutGroup]
2024-11-22T05:34:59,481 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-22T05:34:59,481 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T05:34:59,481 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-22T05:34:59,481 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-22T05:34:59,482 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-22T05:34:59,489 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46801 is added to blk_1073741831_1007 (size=1321)
2024-11-22T05:34:59,490 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35947 is added to blk_1073741831_1007 (size=1321)
2024-11-22T05:34:59,490 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-22T05:34:59,491 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363
2024-11-22T05:34:59,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T05:34:59,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-22T05:34:59,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-22T05:34:59,514 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35947 is added to blk_1073741832_1008 (size=32)
2024-11-22T05:34:59,514 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46801 is added to blk_1073741832_1008 (size=32)
2024-11-22T05:34:59,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:34:59,517 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T05:34:59,518 INFO  [RS:0;82891a793a29:32879 {}] regionserver.HRegionServer(746): ClusterId : 9a354998-d724-47a4-9962-bec68dd1d92f
2024-11-22T05:34:59,518 DEBUG [RS:0;82891a793a29:32879 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-22T05:34:59,519 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T05:34:59,519 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:34:59,520 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:34:59,520 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T05:34:59,522 DEBUG [RS:0;82891a793a29:32879 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-22T05:34:59,522 DEBUG [RS:0;82891a793a29:32879 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-22T05:34:59,522 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T05:34:59,523 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:34:59,523 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:34:59,523 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T05:34:59,525 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T05:34:59,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:34:59,526 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:34:59,526 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T05:34:59,527 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T05:34:59,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:34:59,528 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:34:59,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T05:34:59,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/data/hbase/meta/1588230740
2024-11-22T05:34:59,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/data/hbase/meta/1588230740
2024-11-22T05:34:59,531 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T05:34:59,531 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T05:34:59,532 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T05:34:59,533 DEBUG [RS:0;82891a793a29:32879 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-22T05:34:59,533 DEBUG [RS:0;82891a793a29:32879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc1f0d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82891a793a29/172.17.0.2:0
2024-11-22T05:34:59,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T05:34:59,541 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:34:59,542 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=721635, jitterRate=-0.08239436149597168}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T05:34:59,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732253699515Initializing all the Stores at 1732253699516 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253699516Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253699517 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253699517Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253699517Cleaning up temporary data from old regions at 1732253699532 (+15 ms)Region opened successfully at 1732253699544 (+12 ms)
2024-11-22T05:34:59,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T05:34:59,544 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T05:34:59,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T05:34:59,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T05:34:59,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T05:34:59,549 DEBUG [RS:0;82891a793a29:32879 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;82891a793a29:32879
2024-11-22T05:34:59,549 INFO  [RS:0;82891a793a29:32879 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-22T05:34:59,549 INFO  [RS:0;82891a793a29:32879 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-22T05:34:59,549 DEBUG [RS:0;82891a793a29:32879 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-22T05:34:59,550 INFO  [RS:0;82891a793a29:32879 {}] regionserver.HRegionServer(2659): reportForDuty to master=82891a793a29,39963,1732253698758 with port=32879, startcode=1732253698958
2024-11-22T05:34:59,550 DEBUG [RS:0;82891a793a29:32879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-22T05:34:59,551 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T05:34:59,551 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732253699544Disabling compacts and flushes for region at 1732253699544Disabling writes for close at 1732253699544Writing region close event to WAL at 1732253699551 (+7 ms)Closed at 1732253699551
2024-11-22T05:34:59,553 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:34:59,553 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-22T05:34:59,553 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-22T05:34:59,554 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48943, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-11-22T05:34:59,554 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39963 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 82891a793a29,32879,1732253698958
2024-11-22T05:34:59,555 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39963 {}] master.ServerManager(517): Registering regionserver=82891a793a29,32879,1732253698958
2024-11-22T05:34:59,555 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T05:34:59,558 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-22T05:34:59,559 DEBUG [RS:0;82891a793a29:32879 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363
2024-11-22T05:34:59,559 DEBUG [RS:0;82891a793a29:32879 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38997
2024-11-22T05:34:59,559 DEBUG [RS:0;82891a793a29:32879 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-22T05:34:59,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T05:34:59,568 DEBUG [RS:0;82891a793a29:32879 {}] zookeeper.ZKUtil(111): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/82891a793a29,32879,1732253698958
2024-11-22T05:34:59,568 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [82891a793a29,32879,1732253698958]
2024-11-22T05:34:59,568 WARN  [RS:0;82891a793a29:32879 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T05:34:59,568 INFO  [RS:0;82891a793a29:32879 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:34:59,569 DEBUG [RS:0;82891a793a29:32879 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/WALs/82891a793a29,32879,1732253698958
2024-11-22T05:34:59,573 INFO  [RS:0;82891a793a29:32879 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-22T05:34:59,576 INFO  [RS:0;82891a793a29:32879 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-22T05:34:59,580 INFO  [RS:0;82891a793a29:32879 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T05:34:59,581 INFO  [RS:0;82891a793a29:32879 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:34:59,581 INFO  [RS:0;82891a793a29:32879 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-22T05:34:59,583 INFO  [RS:0;82891a793a29:32879 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-22T05:34:59,583 INFO  [RS:0;82891a793a29:32879 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-22T05:34:59,583 DEBUG [RS:0;82891a793a29:32879 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:34:59,584 DEBUG [RS:0;82891a793a29:32879 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:34:59,584 DEBUG [RS:0;82891a793a29:32879 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:34:59,584 DEBUG [RS:0;82891a793a29:32879 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:34:59,584 DEBUG [RS:0;82891a793a29:32879 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:34:59,584 DEBUG [RS:0;82891a793a29:32879 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/82891a793a29:0, corePoolSize=2, maxPoolSize=2
2024-11-22T05:34:59,584 DEBUG [RS:0;82891a793a29:32879 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:34:59,584 DEBUG [RS:0;82891a793a29:32879 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:34:59,584 DEBUG [RS:0;82891a793a29:32879 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:34:59,585 DEBUG [RS:0;82891a793a29:32879 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:34:59,585 DEBUG [RS:0;82891a793a29:32879 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:34:59,585 DEBUG [RS:0;82891a793a29:32879 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:34:59,585 DEBUG [RS:0;82891a793a29:32879 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/82891a793a29:0, corePoolSize=3, maxPoolSize=3
2024-11-22T05:34:59,585 DEBUG [RS:0;82891a793a29:32879 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0, corePoolSize=3, maxPoolSize=3
2024-11-22T05:34:59,590 INFO  [RS:0;82891a793a29:32879 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T05:34:59,590 INFO  [RS:0;82891a793a29:32879 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T05:34:59,590 INFO  [RS:0;82891a793a29:32879 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:34:59,590 INFO  [RS:0;82891a793a29:32879 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-22T05:34:59,590 INFO  [RS:0;82891a793a29:32879 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-22T05:34:59,590 INFO  [RS:0;82891a793a29:32879 {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,32879,1732253698958-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T05:34:59,609 INFO  [RS:0;82891a793a29:32879 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-22T05:34:59,609 INFO  [RS:0;82891a793a29:32879 {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,32879,1732253698958-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:34:59,609 INFO  [RS:0;82891a793a29:32879 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:34:59,610 INFO  [RS:0;82891a793a29:32879 {}] regionserver.Replication(171): 82891a793a29,32879,1732253698958 started
2024-11-22T05:34:59,630 INFO  [RS:0;82891a793a29:32879 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:34:59,630 INFO  [RS:0;82891a793a29:32879 {}] regionserver.HRegionServer(1482): Serving as 82891a793a29,32879,1732253698958, RpcServer on 82891a793a29/172.17.0.2:32879, sessionid=0x1016109c65b0001
2024-11-22T05:34:59,630 DEBUG [RS:0;82891a793a29:32879 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-22T05:34:59,630 DEBUG [RS:0;82891a793a29:32879 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 82891a793a29,32879,1732253698958
2024-11-22T05:34:59,630 DEBUG [RS:0;82891a793a29:32879 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82891a793a29,32879,1732253698958'
2024-11-22T05:34:59,630 DEBUG [RS:0;82891a793a29:32879 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-22T05:34:59,631 DEBUG [RS:0;82891a793a29:32879 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-22T05:34:59,632 DEBUG [RS:0;82891a793a29:32879 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-22T05:34:59,632 DEBUG [RS:0;82891a793a29:32879 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-22T05:34:59,632 DEBUG [RS:0;82891a793a29:32879 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 82891a793a29,32879,1732253698958
2024-11-22T05:34:59,632 DEBUG [RS:0;82891a793a29:32879 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82891a793a29,32879,1732253698958'
2024-11-22T05:34:59,632 DEBUG [RS:0;82891a793a29:32879 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-22T05:34:59,632 DEBUG [RS:0;82891a793a29:32879 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-22T05:34:59,633 DEBUG [RS:0;82891a793a29:32879 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-22T05:34:59,633 INFO  [RS:0;82891a793a29:32879 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-22T05:34:59,633 INFO  [RS:0;82891a793a29:32879 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-22T05:34:59,708 WARN  [82891a793a29:39963 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-22T05:34:59,736 INFO  [RS:0;82891a793a29:32879 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C32879%2C1732253698958, suffix=, logDir=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/WALs/82891a793a29,32879,1732253698958, archiveDir=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/oldWALs, maxLogs=32
2024-11-22T05:34:59,738 INFO  [RS:0;82891a793a29:32879 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C32879%2C1732253698958.1732253699738
2024-11-22T05:34:59,752 INFO  [RS:0;82891a793a29:32879 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/WALs/82891a793a29,32879,1732253698958/82891a793a29%2C32879%2C1732253698958.1732253699738
2024-11-22T05:34:59,754 DEBUG [RS:0;82891a793a29:32879 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42275:42275),(127.0.0.1/127.0.0.1:42897:42897)]
2024-11-22T05:34:59,958 DEBUG [82891a793a29:39963 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-22T05:34:59,959 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=82891a793a29,32879,1732253698958
2024-11-22T05:34:59,961 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 82891a793a29,32879,1732253698958, state=OPENING
2024-11-22T05:35:00,005 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-22T05:35:00,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:00,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:00,017 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T05:35:00,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:35:00,017 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=82891a793a29,32879,1732253698958}]
2024-11-22T05:35:00,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:35:00,171 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-22T05:35:00,173 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57333, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-22T05:35:00,179 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-22T05:35:00,179 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:35:00,182 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C32879%2C1732253698958.meta, suffix=.meta, logDir=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/WALs/82891a793a29,32879,1732253698958, archiveDir=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/oldWALs, maxLogs=32
2024-11-22T05:35:00,185 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C32879%2C1732253698958.meta.1732253700184.meta
2024-11-22T05:35:00,194 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/WALs/82891a793a29,32879,1732253698958/82891a793a29%2C32879%2C1732253698958.meta.1732253700184.meta
2024-11-22T05:35:00,197 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42897:42897),(127.0.0.1/127.0.0.1:42275:42275)]
2024-11-22T05:35:00,202 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:35:00,202 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-22T05:35:00,202 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-22T05:35:00,203 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-22T05:35:00,203 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-22T05:35:00,203 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:35:00,203 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-22T05:35:00,203 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-22T05:35:00,205 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T05:35:00,206 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T05:35:00,207 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:00,207 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:00,208 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T05:35:00,209 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T05:35:00,209 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:00,210 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:00,210 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T05:35:00,212 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T05:35:00,212 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:00,213 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:00,213 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T05:35:00,214 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T05:35:00,214 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:00,215 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:00,216 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T05:35:00,217 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/data/hbase/meta/1588230740
2024-11-22T05:35:00,219 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/data/hbase/meta/1588230740
2024-11-22T05:35:00,221 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T05:35:00,221 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T05:35:00,222 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T05:35:00,225 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T05:35:00,226 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=841979, jitterRate=0.07063205540180206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T05:35:00,226 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-22T05:35:00,228 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732253700203Writing region info on filesystem at 1732253700203Initializing all the Stores at 1732253700205 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253700205Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253700205Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253700205Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253700205Cleaning up temporary data from old regions at 1732253700221 (+16 ms)Running coprocessor post-open hooks at 1732253700227 (+6 ms)Region opened successfully at 1732253700228 (+1 ms)
2024-11-22T05:35:00,230 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732253700170
2024-11-22T05:35:00,234 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-22T05:35:00,234 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-22T05:35:00,236 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=82891a793a29,32879,1732253698958
2024-11-22T05:35:00,238 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 82891a793a29,32879,1732253698958, state=OPEN
2024-11-22T05:35:00,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T05:35:00,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T05:35:00,277 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=82891a793a29,32879,1732253698958
2024-11-22T05:35:00,277 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:35:00,277 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:35:00,280 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:00,283 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-22T05:35:00,283 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=82891a793a29,32879,1732253698958 in 260 msec
2024-11-22T05:35:00,286 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:00,288 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-22T05:35:00,288 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 730 msec
2024-11-22T05:35:00,289 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:35:00,289 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-22T05:35:00,292 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T05:35:00,292 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82891a793a29,32879,1732253698958, seqNum=-1]
2024-11-22T05:35:00,292 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T05:35:00,294 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48181, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T05:35:00,306 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 830 msec
2024-11-22T05:35:00,306 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732253700306, completionTime=-1
2024-11-22T05:35:00,306 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-22T05:35:00,307 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-22T05:35:00,309 INFO  [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-22T05:35:00,310 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732253760309
2024-11-22T05:35:00,310 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732253820310
2024-11-22T05:35:00,310 INFO  [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-22T05:35:00,310 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,39963,1732253698758-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:00,311 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,39963,1732253698758-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:00,311 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,39963,1732253698758-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:00,311 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-82891a793a29:39963, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:00,311 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:00,311 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:00,315 DEBUG [master/82891a793a29:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-22T05:35:00,319 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.256sec
2024-11-22T05:35:00,319 INFO  [master/82891a793a29:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-22T05:35:00,319 INFO  [master/82891a793a29:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-22T05:35:00,320 INFO  [master/82891a793a29:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-22T05:35:00,320 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-22T05:35:00,320 INFO  [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-22T05:35:00,320 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,39963,1732253698758-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T05:35:00,320 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,39963,1732253698758-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-22T05:35:00,324 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-22T05:35:00,324 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-22T05:35:00,324 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,39963,1732253698758-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:00,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd69191, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:35:00,419 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82891a793a29,39963,-1 for getting cluster id
2024-11-22T05:35:00,419 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-22T05:35:00,431 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a354998-d724-47a4-9962-bec68dd1d92f'
2024-11-22T05:35:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-22T05:35:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a354998-d724-47a4-9962-bec68dd1d92f"

2024-11-22T05:35:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71fba55e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:35:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82891a793a29,39963,-1]
2024-11-22T05:35:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-22T05:35:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:35:00,450 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-22T05:35:00,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e299140, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:35:00,452 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T05:35:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82891a793a29,32879,1732253698958, seqNum=-1]
2024-11-22T05:35:00,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T05:35:00,457 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T05:35:00,459 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=82891a793a29,39963,1732253698758
2024-11-22T05:35:00,460 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:00,464 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-22T05:35:00,464 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-22T05:35:00,464 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T05:35:00,464 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:35:00,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:35:00,465 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-22T05:35:00,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:35:00,465 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-22T05:35:00,465 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=650663950, stopped=false
2024-11-22T05:35:00,465 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=82891a793a29,39963,1732253698758
2024-11-22T05:35:00,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T05:35:00,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:00,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T05:35:00,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:00,480 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T05:35:00,480 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:35:00,481 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:35:00,482 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T05:35:00,482 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:35:00,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:35:00,483 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '82891a793a29,32879,1732253698958' *****
2024-11-22T05:35:00,483 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-22T05:35:00,483 INFO  [RS:0;82891a793a29:32879 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-22T05:35:00,483 INFO  [RS:0;82891a793a29:32879 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-22T05:35:00,483 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-22T05:35:00,483 INFO  [RS:0;82891a793a29:32879 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-22T05:35:00,483 INFO  [RS:0;82891a793a29:32879 {}] regionserver.HRegionServer(959): stopping server 82891a793a29,32879,1732253698958
2024-11-22T05:35:00,483 INFO  [RS:0;82891a793a29:32879 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T05:35:00,484 INFO  [RS:0;82891a793a29:32879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;82891a793a29:32879.
2024-11-22T05:35:00,484 DEBUG [RS:0;82891a793a29:32879 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:35:00,484 DEBUG [RS:0;82891a793a29:32879 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:35:00,484 INFO  [RS:0;82891a793a29:32879 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-22T05:35:00,484 INFO  [RS:0;82891a793a29:32879 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-22T05:35:00,484 INFO  [RS:0;82891a793a29:32879 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-22T05:35:00,484 INFO  [RS:0;82891a793a29:32879 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-22T05:35:00,484 INFO  [RS:0;82891a793a29:32879 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-22T05:35:00,484 DEBUG [RS:0;82891a793a29:32879 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-22T05:35:00,484 DEBUG [RS:0;82891a793a29:32879 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-22T05:35:00,484 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T05:35:00,484 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T05:35:00,485 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T05:35:00,485 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T05:35:00,485 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T05:35:00,485 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-22T05:35:00,510 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/data/hbase/meta/1588230740/.tmp/ns/2f2fa7fae23444eaa44b5806a1be5f8c is 43, key is default/ns:d/1732253700295/Put/seqid=0
2024-11-22T05:35:00,517 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46801 is added to blk_1073741835_1011 (size=5153)
2024-11-22T05:35:00,518 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35947 is added to blk_1073741835_1011 (size=5153)
2024-11-22T05:35:00,518 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/data/hbase/meta/1588230740/.tmp/ns/2f2fa7fae23444eaa44b5806a1be5f8c
2024-11-22T05:35:00,531 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/data/hbase/meta/1588230740/.tmp/ns/2f2fa7fae23444eaa44b5806a1be5f8c as hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/data/hbase/meta/1588230740/ns/2f2fa7fae23444eaa44b5806a1be5f8c
2024-11-22T05:35:00,543 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/data/hbase/meta/1588230740/ns/2f2fa7fae23444eaa44b5806a1be5f8c, entries=2, sequenceid=6, filesize=5.0 K
2024-11-22T05:35:00,545 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 59ms, sequenceid=6, compaction requested=false
2024-11-22T05:35:00,545 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-22T05:35:00,553 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-22T05:35:00,554 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T05:35:00,554 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T05:35:00,554 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732253700484Running coprocessor pre-close hooks at 1732253700484Disabling compacts and flushes for region at 1732253700484Disabling writes for close at 1732253700485 (+1 ms)Obtaining lock to block concurrent updates at 1732253700485Preparing flush snapshotting stores in 1588230740 at 1732253700485Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732253700485Flushing stores of hbase:meta,,1.1588230740 at 1732253700486 (+1 ms)Flushing 1588230740/ns: creating writer at 1732253700487 (+1 ms)Flushing 1588230740/ns: appending metadata at 1732253700509 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1732253700509Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3abb4247: reopening flushed file at 1732253700528 (+19 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 59ms, sequenceid=6, compaction requested=false at 1732253700545 (+17 ms)Writing region close event to WAL at 1732253700547 (+2 ms)Running coprocessor post-close hooks at 1732253700554 (+7 ms)Closed at 1732253700554
2024-11-22T05:35:00,554 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-22T05:35:00,644 INFO  [regionserver/82891a793a29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-22T05:35:00,645 INFO  [regionserver/82891a793a29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-22T05:35:00,685 INFO  [RS:0;82891a793a29:32879 {}] regionserver.HRegionServer(976): stopping server 82891a793a29,32879,1732253698958; all regions closed.
2024-11-22T05:35:00,685 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:00,685 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:00,685 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:00,686 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:00,686 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:00,688 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35947 is added to blk_1073741834_1010 (size=1152)
2024-11-22T05:35:00,689 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46801 is added to blk_1073741834_1010 (size=1152)
2024-11-22T05:35:00,694 DEBUG [RS:0;82891a793a29:32879 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/oldWALs
2024-11-22T05:35:00,695 INFO  [RS:0;82891a793a29:32879 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 82891a793a29%2C32879%2C1732253698958.meta:.meta(num 1732253700184)
2024-11-22T05:35:00,695 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:00,695 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:00,695 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:00,696 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:00,696 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:00,704 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46801 is added to blk_1073741833_1009 (size=93)
2024-11-22T05:35:00,705 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35947 is added to blk_1073741833_1009 (size=93)
2024-11-22T05:35:00,809 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T05:35:00,816 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:00,836 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:01,102 DEBUG [RS:0;82891a793a29:32879 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/oldWALs
2024-11-22T05:35:01,102 INFO  [RS:0;82891a793a29:32879 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 82891a793a29%2C32879%2C1732253698958:(num 1732253699738)
2024-11-22T05:35:01,102 DEBUG [RS:0;82891a793a29:32879 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:35:01,102 INFO  [RS:0;82891a793a29:32879 {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T05:35:01,102 INFO  [RS:0;82891a793a29:32879 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T05:35:01,102 INFO  [RS:0;82891a793a29:32879 {}] hbase.ChoreService(370): Chore service for: regionserver/82891a793a29:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-22T05:35:01,103 INFO  [RS:0;82891a793a29:32879 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T05:35:01,103 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T05:35:01,103 INFO  [RS:0;82891a793a29:32879 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32879
2024-11-22T05:35:01,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/82891a793a29,32879,1732253698958
2024-11-22T05:35:01,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T05:35:01,153 INFO  [RS:0;82891a793a29:32879 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T05:35:01,163 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [82891a793a29,32879,1732253698958]
2024-11-22T05:35:01,174 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/82891a793a29,32879,1732253698958 already deleted, retry=false
2024-11-22T05:35:01,174 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 82891a793a29,32879,1732253698958 expired; onlineServers=0
2024-11-22T05:35:01,174 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '82891a793a29,39963,1732253698758' *****
2024-11-22T05:35:01,174 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-22T05:35:01,174 INFO  [M:0;82891a793a29:39963 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T05:35:01,174 INFO  [M:0;82891a793a29:39963 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T05:35:01,174 DEBUG [M:0;82891a793a29:39963 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-22T05:35:01,175 DEBUG [M:0;82891a793a29:39963 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-22T05:35:01,175 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-22T05:35:01,175 DEBUG [master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253699481 {}] cleaner.HFileCleaner(306): Exit Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253699481,5,FailOnTimeoutGroup]
2024-11-22T05:35:01,175 DEBUG [master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253699481 {}] cleaner.HFileCleaner(306): Exit Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253699481,5,FailOnTimeoutGroup]
2024-11-22T05:35:01,175 INFO  [M:0;82891a793a29:39963 {}] hbase.ChoreService(370): Chore service for: master/82891a793a29:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-22T05:35:01,175 INFO  [M:0;82891a793a29:39963 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T05:35:01,175 DEBUG [M:0;82891a793a29:39963 {}] master.HMaster(1795): Stopping service threads
2024-11-22T05:35:01,175 INFO  [M:0;82891a793a29:39963 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-22T05:35:01,175 INFO  [M:0;82891a793a29:39963 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T05:35:01,175 INFO  [M:0;82891a793a29:39963 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-22T05:35:01,175 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-22T05:35:01,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-22T05:35:01,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:01,185 DEBUG [M:0;82891a793a29:39963 {}] zookeeper.ZKUtil(347): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-22T05:35:01,185 WARN  [M:0;82891a793a29:39963 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-22T05:35:01,186 INFO  [M:0;82891a793a29:39963 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/.lastflushedseqids
2024-11-22T05:35:01,193 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46801 is added to blk_1073741836_1012 (size=99)
2024-11-22T05:35:01,194 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35947 is added to blk_1073741836_1012 (size=99)
2024-11-22T05:35:01,194 INFO  [M:0;82891a793a29:39963 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-22T05:35:01,195 INFO  [M:0;82891a793a29:39963 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-22T05:35:01,195 DEBUG [M:0;82891a793a29:39963 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T05:35:01,195 INFO  [M:0;82891a793a29:39963 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:35:01,195 DEBUG [M:0;82891a793a29:39963 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:35:01,195 DEBUG [M:0;82891a793a29:39963 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T05:35:01,195 DEBUG [M:0;82891a793a29:39963 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:35:01,195 INFO  [M:0;82891a793a29:39963 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-22T05:35:01,217 DEBUG [M:0;82891a793a29:39963 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b7f49d9e7e794ec597fe51c80dbfc12a is 82, key is hbase:meta,,1/info:regioninfo/1732253700235/Put/seqid=0
2024-11-22T05:35:01,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46801 is added to blk_1073741837_1013 (size=5672)
2024-11-22T05:35:01,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35947 is added to blk_1073741837_1013 (size=5672)
2024-11-22T05:35:01,224 INFO  [M:0;82891a793a29:39963 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b7f49d9e7e794ec597fe51c80dbfc12a
2024-11-22T05:35:01,251 DEBUG [M:0;82891a793a29:39963 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/478617feba0749b8ba95079e035a7789 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732253700305/Put/seqid=0
2024-11-22T05:35:01,258 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35947 is added to blk_1073741838_1014 (size=5275)
2024-11-22T05:35:01,258 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46801 is added to blk_1073741838_1014 (size=5275)
2024-11-22T05:35:01,263 INFO  [RS:0;82891a793a29:32879 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T05:35:01,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:35:01,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32879-0x1016109c65b0001, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:35:01,264 INFO  [RS:0;82891a793a29:32879 {}] regionserver.HRegionServer(1031): Exiting; stopping=82891a793a29,32879,1732253698958; zookeeper connection closed.
2024-11-22T05:35:01,264 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6bc187f6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6bc187f6
2024-11-22T05:35:01,264 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-22T05:35:01,591 INFO  [regionserver/82891a793a29:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T05:35:01,659 INFO  [M:0;82891a793a29:39963 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/478617feba0749b8ba95079e035a7789
2024-11-22T05:35:01,684 DEBUG [M:0;82891a793a29:39963 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1a94f618ed7046fe8bd57b64599f568a is 69, key is 82891a793a29,32879,1732253698958/rs:state/1732253699555/Put/seqid=0
2024-11-22T05:35:01,694 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35947 is added to blk_1073741839_1015 (size=5156)
2024-11-22T05:35:01,695 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46801 is added to blk_1073741839_1015 (size=5156)
2024-11-22T05:35:01,695 INFO  [M:0;82891a793a29:39963 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1a94f618ed7046fe8bd57b64599f568a
2024-11-22T05:35:01,727 DEBUG [M:0;82891a793a29:39963 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6431b8c9cdb34310959a39210dc121a3 is 52, key is load_balancer_on/state:d/1732253700462/Put/seqid=0
2024-11-22T05:35:01,734 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46801 is added to blk_1073741840_1016 (size=5056)
2024-11-22T05:35:01,734 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35947 is added to blk_1073741840_1016 (size=5056)
2024-11-22T05:35:01,735 INFO  [M:0;82891a793a29:39963 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6431b8c9cdb34310959a39210dc121a3
2024-11-22T05:35:01,745 DEBUG [M:0;82891a793a29:39963 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b7f49d9e7e794ec597fe51c80dbfc12a as hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b7f49d9e7e794ec597fe51c80dbfc12a
2024-11-22T05:35:01,754 INFO  [M:0;82891a793a29:39963 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b7f49d9e7e794ec597fe51c80dbfc12a, entries=8, sequenceid=29, filesize=5.5 K
2024-11-22T05:35:01,755 DEBUG [M:0;82891a793a29:39963 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/478617feba0749b8ba95079e035a7789 as hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/478617feba0749b8ba95079e035a7789
2024-11-22T05:35:01,763 INFO  [M:0;82891a793a29:39963 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/478617feba0749b8ba95079e035a7789, entries=3, sequenceid=29, filesize=5.2 K
2024-11-22T05:35:01,765 DEBUG [M:0;82891a793a29:39963 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1a94f618ed7046fe8bd57b64599f568a as hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1a94f618ed7046fe8bd57b64599f568a
2024-11-22T05:35:01,772 INFO  [M:0;82891a793a29:39963 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1a94f618ed7046fe8bd57b64599f568a, entries=1, sequenceid=29, filesize=5.0 K
2024-11-22T05:35:01,774 DEBUG [M:0;82891a793a29:39963 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6431b8c9cdb34310959a39210dc121a3 as hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6431b8c9cdb34310959a39210dc121a3
2024-11-22T05:35:01,782 INFO  [M:0;82891a793a29:39963 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38997/user/jenkins/test-data/6b3098a7-5e18-1e0f-8f94-e65fbe001363/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6431b8c9cdb34310959a39210dc121a3, entries=1, sequenceid=29, filesize=4.9 K
2024-11-22T05:35:01,784 INFO  [M:0;82891a793a29:39963 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 589ms, sequenceid=29, compaction requested=false
2024-11-22T05:35:01,786 INFO  [M:0;82891a793a29:39963 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:35:01,786 DEBUG [M:0;82891a793a29:39963 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732253701195Disabling compacts and flushes for region at 1732253701195Disabling writes for close at 1732253701195Obtaining lock to block concurrent updates at 1732253701195Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732253701195Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732253701196 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732253701197 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732253701197Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732253701217 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732253701217Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732253701232 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732253701251 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732253701251Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732253701667 (+416 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732253701683 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732253701684 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732253701703 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732253701726 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732253701726Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32ac991a: reopening flushed file at 1732253701744 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76f38bd9: reopening flushed file at 1732253701754 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7720d9c4: reopening flushed file at 1732253701764 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cdfb48e: reopening flushed file at 1732253701773 (+9 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 589ms, sequenceid=29, compaction requested=false at 1732253701784 (+11 ms)Writing region close event to WAL at 1732253701786 (+2 ms)Closed at 1732253701786
2024-11-22T05:35:01,786 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:01,787 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:01,787 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:01,787 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:01,787 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:01,789 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35947 is added to blk_1073741830_1006 (size=10311)
2024-11-22T05:35:01,789 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46801 is added to blk_1073741830_1006 (size=10311)
2024-11-22T05:35:01,792 INFO  [M:0;82891a793a29:39963 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-22T05:35:01,792 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T05:35:01,792 INFO  [M:0;82891a793a29:39963 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39963
2024-11-22T05:35:01,792 INFO  [M:0;82891a793a29:39963 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T05:35:01,937 INFO  [M:0;82891a793a29:39963 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T05:35:01,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:35:01,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39963-0x1016109c65b0000, quorum=127.0.0.1:59969, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:35:01,940 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c17f9e5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:35:01,940 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@669fbf01{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:35:01,940 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:35:01,940 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d5e070a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:35:01,941 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a742c1f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/hadoop.log.dir/,STOPPED}
2024-11-22T05:35:01,942 WARN  [BP-1370325539-172.17.0.2-1732253696356 heartbeating to localhost/127.0.0.1:38997 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:35:01,942 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:35:01,942 WARN  [BP-1370325539-172.17.0.2-1732253696356 heartbeating to localhost/127.0.0.1:38997 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1370325539-172.17.0.2-1732253696356 (Datanode Uuid 8ead25df-fb9e-4f76-9e9f-fe7257afe5ac) service to localhost/127.0.0.1:38997
2024-11-22T05:35:01,942 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:35:01,943 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/cluster_b7ed9aed-9d3c-9602-ce42-fac914215dcb/data/data3/current/BP-1370325539-172.17.0.2-1732253696356 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:35:01,943 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/cluster_b7ed9aed-9d3c-9602-ce42-fac914215dcb/data/data4/current/BP-1370325539-172.17.0.2-1732253696356 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:35:01,943 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:35:01,946 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@66b9e907{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:35:01,946 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@51f8cb4f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:35:01,947 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:35:01,947 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1aa9c156{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:35:01,947 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b58749b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/hadoop.log.dir/,STOPPED}
2024-11-22T05:35:01,948 WARN  [BP-1370325539-172.17.0.2-1732253696356 heartbeating to localhost/127.0.0.1:38997 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:35:01,948 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:35:01,948 WARN  [BP-1370325539-172.17.0.2-1732253696356 heartbeating to localhost/127.0.0.1:38997 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1370325539-172.17.0.2-1732253696356 (Datanode Uuid 285f8640-8b38-4fc0-ae10-71cd72d55cf8) service to localhost/127.0.0.1:38997
2024-11-22T05:35:01,948 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:35:01,949 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/cluster_b7ed9aed-9d3c-9602-ce42-fac914215dcb/data/data1/current/BP-1370325539-172.17.0.2-1732253696356 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:35:01,949 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/cluster_b7ed9aed-9d3c-9602-ce42-fac914215dcb/data/data2/current/BP-1370325539-172.17.0.2-1732253696356 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:35:01,949 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:35:01,955 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2a7d11e5{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T05:35:01,956 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5d4bdc00{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:35:01,956 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:35:01,956 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@641eaf99{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:35:01,956 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@737d6115{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/hadoop.log.dir/,STOPPED}
2024-11-22T05:35:01,963 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-22T05:35:01,988 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-22T05:35:01,988 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-22T05:35:01,989 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/hadoop.log.dir so I do NOT create it in target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff
2024-11-22T05:35:01,989 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a866cb63-c2fa-3fcb-6ff0-a5453a22ea6b/hadoop.tmp.dir so I do NOT create it in target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff
2024-11-22T05:35:01,989 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2, deleteOnExit=true
2024-11-22T05:35:01,989 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-22T05:35:01,989 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/test.cache.data in system properties and HBase conf
2024-11-22T05:35:01,989 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.tmp.dir in system properties and HBase conf
2024-11-22T05:35:01,989 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.log.dir in system properties and HBase conf
2024-11-22T05:35:01,989 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-22T05:35:01,990 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-22T05:35:01,990 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-22T05:35:01,990 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-22T05:35:01,990 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-22T05:35:01,990 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-22T05:35:01,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-22T05:35:01,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T05:35:01,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-22T05:35:01,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-22T05:35:01,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T05:35:01,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T05:35:01,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-22T05:35:01,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/nfs.dump.dir in system properties and HBase conf
2024-11-22T05:35:01,992 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/java.io.tmpdir in system properties and HBase conf
2024-11-22T05:35:01,992 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T05:35:01,992 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-22T05:35:01,992 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-22T05:35:02,010 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T05:35:02,350 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:35:02,363 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:35:02,373 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:35:02,373 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:35:02,373 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T05:35:02,374 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:35:02,374 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ab5393f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:35:02,375 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4ac253d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:35:02,482 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2295376c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/java.io.tmpdir/jetty-localhost-44367-hadoop-hdfs-3_4_1-tests_jar-_-any-10133052941168848438/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T05:35:02,482 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@54adbc26{HTTP/1.1, (http/1.1)}{localhost:44367}
2024-11-22T05:35:02,483 INFO  [Time-limited test {}] server.Server(415): Started @109740ms
2024-11-22T05:35:02,501 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T05:35:02,769 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:35:02,773 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:35:02,774 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:35:02,774 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:35:02,774 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:35:02,775 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@372d60ec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:35:02,775 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7a18c5e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:35:02,881 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bba803f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/java.io.tmpdir/jetty-localhost-35729-hadoop-hdfs-3_4_1-tests_jar-_-any-9949676766054597138/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:35:02,882 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7629a449{HTTP/1.1, (http/1.1)}{localhost:35729}
2024-11-22T05:35:02,882 INFO  [Time-limited test {}] server.Server(415): Started @110139ms
2024-11-22T05:35:02,884 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:35:02,974 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:35:02,979 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:35:02,981 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:35:02,981 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:35:02,981 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T05:35:02,981 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@217a95d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:35:02,982 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2c64d82b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:35:03,091 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3efce601{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/java.io.tmpdir/jetty-localhost-38097-hadoop-hdfs-3_4_1-tests_jar-_-any-2293186374874945268/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:35:03,091 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@e79a3d4{HTTP/1.1, (http/1.1)}{localhost:38097}
2024-11-22T05:35:03,091 INFO  [Time-limited test {}] server.Server(415): Started @110349ms
2024-11-22T05:35:03,093 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:35:04,307 WARN  [Thread-672 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data1/current/BP-860069939-172.17.0.2-1732253702022/current, will proceed with Du for space computation calculation, 
2024-11-22T05:35:04,307 WARN  [Thread-673 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data2/current/BP-860069939-172.17.0.2-1732253702022/current, will proceed with Du for space computation calculation, 
2024-11-22T05:35:04,330 WARN  [Thread-636 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:35:04,333 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x70ccbe55c217165f with lease ID 0x2f19ab655b3c7ebc: Processing first storage report for DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e from datanode DatanodeRegistration(127.0.0.1:41019, datanodeUuid=4b522218-4cbe-42f6-948e-e6c30b984188, infoPort=34441, infoSecurePort=0, ipcPort=34853, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022)
2024-11-22T05:35:04,333 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x70ccbe55c217165f with lease ID 0x2f19ab655b3c7ebc: from storage DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e node DatanodeRegistration(127.0.0.1:41019, datanodeUuid=4b522218-4cbe-42f6-948e-e6c30b984188, infoPort=34441, infoSecurePort=0, ipcPort=34853, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:35:04,333 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x70ccbe55c217165f with lease ID 0x2f19ab655b3c7ebc: Processing first storage report for DS-23f80fdb-50d7-45b8-a02f-2342f6684c0c from datanode DatanodeRegistration(127.0.0.1:41019, datanodeUuid=4b522218-4cbe-42f6-948e-e6c30b984188, infoPort=34441, infoSecurePort=0, ipcPort=34853, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022)
2024-11-22T05:35:04,333 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x70ccbe55c217165f with lease ID 0x2f19ab655b3c7ebc: from storage DS-23f80fdb-50d7-45b8-a02f-2342f6684c0c node DatanodeRegistration(127.0.0.1:41019, datanodeUuid=4b522218-4cbe-42f6-948e-e6c30b984188, infoPort=34441, infoSecurePort=0, ipcPort=34853, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-22T05:35:04,416 WARN  [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data4/current/BP-860069939-172.17.0.2-1732253702022/current, will proceed with Du for space computation calculation, 
2024-11-22T05:35:04,416 WARN  [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data3/current/BP-860069939-172.17.0.2-1732253702022/current, will proceed with Du for space computation calculation, 
2024-11-22T05:35:04,444 WARN  [Thread-659 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:35:04,453 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa1b5f1c5fd058e37 with lease ID 0x2f19ab655b3c7ebd: Processing first storage report for DS-af01850a-5d72-4c8b-a217-d2df8ee2427d from datanode DatanodeRegistration(127.0.0.1:41959, datanodeUuid=7022ae67-a4d8-42d1-b7a4-82fbd184c97e, infoPort=37311, infoSecurePort=0, ipcPort=42547, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022)
2024-11-22T05:35:04,453 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa1b5f1c5fd058e37 with lease ID 0x2f19ab655b3c7ebd: from storage DS-af01850a-5d72-4c8b-a217-d2df8ee2427d node DatanodeRegistration(127.0.0.1:41959, datanodeUuid=7022ae67-a4d8-42d1-b7a4-82fbd184c97e, infoPort=37311, infoSecurePort=0, ipcPort=42547, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-22T05:35:04,453 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa1b5f1c5fd058e37 with lease ID 0x2f19ab655b3c7ebd: Processing first storage report for DS-4f178493-56f5-4235-9403-d0648c301e42 from datanode DatanodeRegistration(127.0.0.1:41959, datanodeUuid=7022ae67-a4d8-42d1-b7a4-82fbd184c97e, infoPort=37311, infoSecurePort=0, ipcPort=42547, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022)
2024-11-22T05:35:04,453 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa1b5f1c5fd058e37 with lease ID 0x2f19ab655b3c7ebd: from storage DS-4f178493-56f5-4235-9403-d0648c301e42 node DatanodeRegistration(127.0.0.1:41959, datanodeUuid=7022ae67-a4d8-42d1-b7a4-82fbd184c97e, infoPort=37311, infoSecurePort=0, ipcPort=42547, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:35:04,463 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff
2024-11-22T05:35:04,475 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/zookeeper_0, clientPort=59485, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-22T05:35:04,476 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59485
2024-11-22T05:35:04,476 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:04,478 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:04,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41019 is added to blk_1073741825_1001 (size=7)
2024-11-22T05:35:04,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741825_1001 (size=7)
2024-11-22T05:35:04,489 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253 with version=8
2024-11-22T05:35:04,489 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/hbase-staging
2024-11-22T05:35:04,492 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/82891a793a29:0 server-side Connection retries=45
2024-11-22T05:35:04,492 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:35:04,492 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T05:35:04,492 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T05:35:04,492 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:35:04,492 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T05:35:04,492 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-22T05:35:04,492 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T05:35:04,493 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36973
2024-11-22T05:35:04,495 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36973 connecting to ZooKeeper ensemble=127.0.0.1:59485
2024-11-22T05:35:04,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:369730x0, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T05:35:04,552 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36973-0x1016109dcbc0000 connected
2024-11-22T05:35:04,637 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:04,639 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:04,642 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:35:04,643 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253, hbase.cluster.distributed=false
2024-11-22T05:35:04,644 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T05:35:04,645 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36973
2024-11-22T05:35:04,645 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36973
2024-11-22T05:35:04,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36973
2024-11-22T05:35:04,656 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36973
2024-11-22T05:35:04,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36973
2024-11-22T05:35:04,679 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/82891a793a29:0 server-side Connection retries=45
2024-11-22T05:35:04,680 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:35:04,680 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T05:35:04,680 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T05:35:04,680 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:35:04,680 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T05:35:04,680 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-22T05:35:04,680 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T05:35:04,681 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35099
2024-11-22T05:35:04,684 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35099 connecting to ZooKeeper ensemble=127.0.0.1:59485
2024-11-22T05:35:04,684 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:04,687 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:04,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:350990x0, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T05:35:04,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:350990x0, quorum=127.0.0.1:59485, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:35:04,703 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-22T05:35:04,708 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35099-0x1016109dcbc0001 connected
2024-11-22T05:35:04,716 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-22T05:35:04,717 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-22T05:35:04,719 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T05:35:04,719 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35099
2024-11-22T05:35:04,720 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35099
2024-11-22T05:35:04,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35099
2024-11-22T05:35:04,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35099
2024-11-22T05:35:04,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35099
2024-11-22T05:35:04,747 DEBUG [M:0;82891a793a29:36973 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;82891a793a29:36973
2024-11-22T05:35:04,747 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/82891a793a29,36973,1732253704491
2024-11-22T05:35:04,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:35:04,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:35:04,753 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/82891a793a29,36973,1732253704491
2024-11-22T05:35:04,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-22T05:35:04,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:04,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:04,764 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-22T05:35:04,764 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/82891a793a29,36973,1732253704491 from backup master directory
2024-11-22T05:35:04,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/82891a793a29,36973,1732253704491
2024-11-22T05:35:04,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:35:04,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:35:04,774 WARN  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T05:35:04,774 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=82891a793a29,36973,1732253704491
2024-11-22T05:35:04,779 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/hbase.id] with ID: 8ac4ac39-ce31-486d-9d9f-1ad34a14fa25
2024-11-22T05:35:04,779 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/.tmp/hbase.id
2024-11-22T05:35:04,789 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41019 is added to blk_1073741826_1002 (size=42)
2024-11-22T05:35:04,790 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741826_1002 (size=42)
2024-11-22T05:35:04,790 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/.tmp/hbase.id]:[hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/hbase.id]
2024-11-22T05:35:04,807 INFO  [master/82891a793a29:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:04,807 INFO  [master/82891a793a29:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-22T05:35:04,810 INFO  [master/82891a793a29:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms.
2024-11-22T05:35:04,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:04,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:04,823 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41019 is added to blk_1073741827_1003 (size=196)
2024-11-22T05:35:04,824 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741827_1003 (size=196)
2024-11-22T05:35:04,825 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T05:35:04,826 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-22T05:35:04,826 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:35:04,838 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41019 is added to blk_1073741828_1004 (size=1189)
2024-11-22T05:35:04,839 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store
2024-11-22T05:35:04,840 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741828_1004 (size=1189)
2024-11-22T05:35:04,847 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741829_1005 (size=34)
2024-11-22T05:35:04,847 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41019 is added to blk_1073741829_1005 (size=34)
2024-11-22T05:35:04,848 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:35:04,848 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T05:35:04,848 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:35:04,848 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:35:04,848 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T05:35:04,848 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:35:04,848 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:35:04,848 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732253704848Disabling compacts and flushes for region at 1732253704848Disabling writes for close at 1732253704848Writing region close event to WAL at 1732253704848Closed at 1732253704848
2024-11-22T05:35:04,849 WARN  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/.initializing
2024-11-22T05:35:04,849 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/WALs/82891a793a29,36973,1732253704491
2024-11-22T05:35:04,853 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C36973%2C1732253704491, suffix=, logDir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/WALs/82891a793a29,36973,1732253704491, archiveDir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/oldWALs, maxLogs=10
2024-11-22T05:35:04,854 INFO  [master/82891a793a29:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C36973%2C1732253704491.1732253704853
2024-11-22T05:35:04,860 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/WALs/82891a793a29,36973,1732253704491/82891a793a29%2C36973%2C1732253704491.1732253704853
2024-11-22T05:35:04,863 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37311:37311),(127.0.0.1/127.0.0.1:34441:34441)]
2024-11-22T05:35:04,864 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:35:04,864 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:35:04,864 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:35:04,864 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:35:04,866 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:35:04,868 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-22T05:35:04,868 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:04,869 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:04,869 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:35:04,871 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-22T05:35:04,871 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:04,872 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:35:04,872 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:35:04,873 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-22T05:35:04,874 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:04,874 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:35:04,874 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:35:04,876 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-22T05:35:04,876 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:04,877 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:35:04,877 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:35:04,878 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:35:04,878 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:35:04,880 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:35:04,880 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:35:04,881 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-22T05:35:04,882 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:35:04,885 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:35:04,885 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=878765, jitterRate=0.11740793287754059}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-22T05:35:04,886 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732253704865Initializing all the Stores at 1732253704866 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253704866Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253704866Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253704866Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253704866Cleaning up temporary data from old regions at 1732253704880 (+14 ms)Region opened successfully at 1732253704886 (+6 ms)
2024-11-22T05:35:04,887 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-22T05:35:04,891 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@460c0c58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82891a793a29/172.17.0.2:0
2024-11-22T05:35:04,892 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-22T05:35:04,892 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-22T05:35:04,892 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-22T05:35:04,892 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-22T05:35:04,893 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-22T05:35:04,893 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-22T05:35:04,894 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-22T05:35:04,896 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-22T05:35:04,897 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-22T05:35:04,900 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-22T05:35:04,900 INFO  [master/82891a793a29:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-22T05:35:04,901 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-22T05:35:04,910 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-22T05:35:04,911 INFO  [master/82891a793a29:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-22T05:35:04,912 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-22T05:35:04,921 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-22T05:35:04,922 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-22T05:35:04,931 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-22T05:35:04,934 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-22T05:35:04,993 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-22T05:35:05,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T05:35:05,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T05:35:05,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:05,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:05,016 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=82891a793a29,36973,1732253704491, sessionid=0x1016109dcbc0000, setting cluster-up flag (Was=false)
2024-11-22T05:35:05,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:05,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:05,068 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-22T05:35:05,070 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=82891a793a29,36973,1732253704491
2024-11-22T05:35:05,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:05,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:05,121 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-22T05:35:05,122 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=82891a793a29,36973,1732253704491
2024-11-22T05:35:05,124 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-22T05:35:05,126 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-22T05:35:05,126 INFO  [master/82891a793a29:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-22T05:35:05,127 INFO  [master/82891a793a29:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-22T05:35:05,127 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 82891a793a29,36973,1732253704491
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-22T05:35:05,129 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:35:05,129 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:35:05,129 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:35:05,130 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:35:05,130 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/82891a793a29:0, corePoolSize=10, maxPoolSize=10
2024-11-22T05:35:05,130 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:05,130 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/82891a793a29:0, corePoolSize=2, maxPoolSize=2
2024-11-22T05:35:05,130 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:05,132 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732253735131
2024-11-22T05:35:05,132 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-22T05:35:05,132 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-22T05:35:05,132 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-22T05:35:05,132 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-22T05:35:05,132 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-22T05:35:05,132 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-22T05:35:05,133 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,133 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:35:05,133 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-22T05:35:05,134 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-22T05:35:05,134 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-22T05:35:05,134 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-22T05:35:05,134 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-22T05:35:05,134 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-22T05:35:05,135 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:05,135 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253705135,5,FailOnTimeoutGroup]
2024-11-22T05:35:05,135 INFO  [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(746): ClusterId : 8ac4ac39-ce31-486d-9d9f-1ad34a14fa25
2024-11-22T05:35:05,135 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253705135,5,FailOnTimeoutGroup]
2024-11-22T05:35:05,135 DEBUG [RS:0;82891a793a29:35099 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-22T05:35:05,135 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,135 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-22T05:35:05,135 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,135 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-22T05:35:05,135 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,143 DEBUG [RS:0;82891a793a29:35099 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-22T05:35:05,143 DEBUG [RS:0;82891a793a29:35099 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-22T05:35:05,153 DEBUG [RS:0;82891a793a29:35099 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-22T05:35:05,154 DEBUG [RS:0;82891a793a29:35099 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62b6da10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82891a793a29/172.17.0.2:0
2024-11-22T05:35:05,160 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41019 is added to blk_1073741831_1007 (size=1321)
2024-11-22T05:35:05,160 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741831_1007 (size=1321)
2024-11-22T05:35:05,161 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-22T05:35:05,161 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253
2024-11-22T05:35:05,171 DEBUG [RS:0;82891a793a29:35099 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;82891a793a29:35099
2024-11-22T05:35:05,171 INFO  [RS:0;82891a793a29:35099 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-22T05:35:05,171 INFO  [RS:0;82891a793a29:35099 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-22T05:35:05,171 DEBUG [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-22T05:35:05,172 INFO  [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(2659): reportForDuty to master=82891a793a29,36973,1732253704491 with port=35099, startcode=1732253704679
2024-11-22T05:35:05,172 DEBUG [RS:0;82891a793a29:35099 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-22T05:35:05,174 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741832_1008 (size=32)
2024-11-22T05:35:05,175 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41019 is added to blk_1073741832_1008 (size=32)
2024-11-22T05:35:05,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:35:05,176 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53899, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-11-22T05:35:05,176 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36973 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 82891a793a29,35099,1732253704679
2024-11-22T05:35:05,177 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36973 {}] master.ServerManager(517): Registering regionserver=82891a793a29,35099,1732253704679
2024-11-22T05:35:05,177 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T05:35:05,179 DEBUG [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253
2024-11-22T05:35:05,179 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T05:35:05,179 DEBUG [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46791
2024-11-22T05:35:05,179 DEBUG [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-22T05:35:05,180 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:05,180 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:05,181 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T05:35:05,182 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T05:35:05,182 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:05,183 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:05,183 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T05:35:05,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T05:35:05,185 DEBUG [RS:0;82891a793a29:35099 {}] zookeeper.ZKUtil(111): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/82891a793a29,35099,1732253704679
2024-11-22T05:35:05,185 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T05:35:05,185 WARN  [RS:0;82891a793a29:35099 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T05:35:05,185 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:05,185 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [82891a793a29,35099,1732253704679]
2024-11-22T05:35:05,185 INFO  [RS:0;82891a793a29:35099 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:35:05,185 DEBUG [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679
2024-11-22T05:35:05,186 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:05,186 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T05:35:05,187 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T05:35:05,187 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:05,188 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:05,188 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T05:35:05,189 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740
2024-11-22T05:35:05,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740
2024-11-22T05:35:05,191 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T05:35:05,191 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T05:35:05,192 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T05:35:05,194 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T05:35:05,196 INFO  [RS:0;82891a793a29:35099 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-22T05:35:05,196 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:35:05,197 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=690780, jitterRate=-0.12162889540195465}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T05:35:05,198 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732253705175Initializing all the Stores at 1732253705176 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253705176Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253705177 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253705177Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253705177Cleaning up temporary data from old regions at 1732253705191 (+14 ms)Region opened successfully at 1732253705198 (+7 ms)
2024-11-22T05:35:05,198 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T05:35:05,198 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T05:35:05,198 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T05:35:05,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T05:35:05,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T05:35:05,199 INFO  [RS:0;82891a793a29:35099 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-22T05:35:05,200 INFO  [RS:0;82891a793a29:35099 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T05:35:05,200 INFO  [RS:0;82891a793a29:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,201 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T05:35:05,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732253705198Disabling compacts and flushes for region at 1732253705198Disabling writes for close at 1732253705199 (+1 ms)Writing region close event to WAL at 1732253705201 (+2 ms)Closed at 1732253705201
2024-11-22T05:35:05,202 INFO  [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-22T05:35:05,203 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:35:05,203 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-22T05:35:05,203 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-22T05:35:05,205 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T05:35:05,207 INFO  [RS:0;82891a793a29:35099 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-22T05:35:05,207 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-22T05:35:05,207 INFO  [RS:0;82891a793a29:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,207 DEBUG [RS:0;82891a793a29:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:05,207 DEBUG [RS:0;82891a793a29:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:05,207 DEBUG [RS:0;82891a793a29:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:05,207 DEBUG [RS:0;82891a793a29:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:05,207 DEBUG [RS:0;82891a793a29:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:05,208 DEBUG [RS:0;82891a793a29:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/82891a793a29:0, corePoolSize=2, maxPoolSize=2
2024-11-22T05:35:05,208 DEBUG [RS:0;82891a793a29:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:05,208 DEBUG [RS:0;82891a793a29:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:05,208 DEBUG [RS:0;82891a793a29:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:05,208 DEBUG [RS:0;82891a793a29:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:05,208 DEBUG [RS:0;82891a793a29:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:05,208 DEBUG [RS:0;82891a793a29:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:05,208 DEBUG [RS:0;82891a793a29:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/82891a793a29:0, corePoolSize=3, maxPoolSize=3
2024-11-22T05:35:05,208 DEBUG [RS:0;82891a793a29:35099 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0, corePoolSize=3, maxPoolSize=3
2024-11-22T05:35:05,216 INFO  [RS:0;82891a793a29:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,216 INFO  [RS:0;82891a793a29:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,216 INFO  [RS:0;82891a793a29:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,216 INFO  [RS:0;82891a793a29:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,216 INFO  [RS:0;82891a793a29:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,217 INFO  [RS:0;82891a793a29:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,35099,1732253704679-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T05:35:05,240 INFO  [RS:0;82891a793a29:35099 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-22T05:35:05,240 INFO  [RS:0;82891a793a29:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,35099,1732253704679-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,240 INFO  [RS:0;82891a793a29:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,241 INFO  [RS:0;82891a793a29:35099 {}] regionserver.Replication(171): 82891a793a29,35099,1732253704679 started
2024-11-22T05:35:05,255 INFO  [RS:0;82891a793a29:35099 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,255 INFO  [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(1482): Serving as 82891a793a29,35099,1732253704679, RpcServer on 82891a793a29/172.17.0.2:35099, sessionid=0x1016109dcbc0001
2024-11-22T05:35:05,255 DEBUG [RS:0;82891a793a29:35099 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-22T05:35:05,255 DEBUG [RS:0;82891a793a29:35099 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 82891a793a29,35099,1732253704679
2024-11-22T05:35:05,255 DEBUG [RS:0;82891a793a29:35099 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82891a793a29,35099,1732253704679'
2024-11-22T05:35:05,256 DEBUG [RS:0;82891a793a29:35099 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-22T05:35:05,256 DEBUG [RS:0;82891a793a29:35099 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-22T05:35:05,257 DEBUG [RS:0;82891a793a29:35099 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-22T05:35:05,257 DEBUG [RS:0;82891a793a29:35099 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-22T05:35:05,257 DEBUG [RS:0;82891a793a29:35099 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 82891a793a29,35099,1732253704679
2024-11-22T05:35:05,257 DEBUG [RS:0;82891a793a29:35099 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82891a793a29,35099,1732253704679'
2024-11-22T05:35:05,257 DEBUG [RS:0;82891a793a29:35099 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-22T05:35:05,257 DEBUG [RS:0;82891a793a29:35099 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-22T05:35:05,257 DEBUG [RS:0;82891a793a29:35099 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-22T05:35:05,257 INFO  [RS:0;82891a793a29:35099 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-22T05:35:05,258 INFO  [RS:0;82891a793a29:35099 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-22T05:35:05,357 WARN  [82891a793a29:36973 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-22T05:35:05,360 INFO  [RS:0;82891a793a29:35099 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C35099%2C1732253704679, suffix=, logDir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679, archiveDir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/oldWALs, maxLogs=32
2024-11-22T05:35:05,361 INFO  [RS:0;82891a793a29:35099 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C35099%2C1732253704679.1732253705361
2024-11-22T05:35:05,393 INFO  [RS:0;82891a793a29:35099 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253705361
2024-11-22T05:35:05,397 DEBUG [RS:0;82891a793a29:35099 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34441:34441),(127.0.0.1/127.0.0.1:37311:37311)]
2024-11-22T05:35:05,607 DEBUG [82891a793a29:36973 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-22T05:35:05,608 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=82891a793a29,35099,1732253704679
2024-11-22T05:35:05,610 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 82891a793a29,35099,1732253704679, state=OPENING
2024-11-22T05:35:05,658 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-22T05:35:05,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:05,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:05,669 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T05:35:05,669 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=82891a793a29,35099,1732253704679}]
2024-11-22T05:35:05,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:35:05,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:35:05,706 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T05:35:05,709 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:05,728 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:05,730 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:05,731 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:05,824 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-22T05:35:05,826 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55711, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-22T05:35:05,831 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-22T05:35:05,831 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:35:05,834 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C35099%2C1732253704679.meta, suffix=.meta, logDir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679, archiveDir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/oldWALs, maxLogs=32
2024-11-22T05:35:05,835 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
2024-11-22T05:35:05,842 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
2024-11-22T05:35:05,843 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37311:37311),(127.0.0.1/127.0.0.1:34441:34441)]
2024-11-22T05:35:05,848 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:35:05,849 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-22T05:35:05,849 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-22T05:35:05,849 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-22T05:35:05,849 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-22T05:35:05,849 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:35:05,849 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-22T05:35:05,849 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-22T05:35:05,851 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T05:35:05,852 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T05:35:05,852 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:05,853 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:05,853 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T05:35:05,854 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T05:35:05,854 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:05,855 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:05,855 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T05:35:05,856 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T05:35:05,856 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:05,856 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:05,857 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T05:35:05,857 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T05:35:05,858 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:05,858 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:05,858 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T05:35:05,859 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740
2024-11-22T05:35:05,861 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740
2024-11-22T05:35:05,862 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T05:35:05,862 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T05:35:05,863 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T05:35:05,864 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T05:35:05,866 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=782162, jitterRate=-0.00543057918548584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T05:35:05,866 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-22T05:35:05,866 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732253705849Writing region info on filesystem at 1732253705849Initializing all the Stores at 1732253705850 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253705850Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253705851 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253705851Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253705851Cleaning up temporary data from old regions at 1732253705862 (+11 ms)Running coprocessor post-open hooks at 1732253705866 (+4 ms)Region opened successfully at 1732253705866
2024-11-22T05:35:05,868 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732253705823
2024-11-22T05:35:05,871 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-22T05:35:05,871 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-22T05:35:05,872 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=82891a793a29,35099,1732253704679
2024-11-22T05:35:05,873 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 82891a793a29,35099,1732253704679, state=OPEN
2024-11-22T05:35:05,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T05:35:05,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T05:35:05,910 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=82891a793a29,35099,1732253704679
2024-11-22T05:35:05,910 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:35:05,910 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:35:05,914 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-22T05:35:05,914 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=82891a793a29,35099,1732253704679 in 241 msec
2024-11-22T05:35:05,917 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-22T05:35:05,917 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 711 msec
2024-11-22T05:35:05,918 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:35:05,918 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-22T05:35:05,920 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T05:35:05,920 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82891a793a29,35099,1732253704679, seqNum=-1]
2024-11-22T05:35:05,920 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T05:35:05,922 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57327, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T05:35:05,928 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 802 msec
2024-11-22T05:35:05,929 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732253705929, completionTime=-1
2024-11-22T05:35:05,929 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-22T05:35:05,929 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-22T05:35:05,931 INFO  [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-22T05:35:05,931 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732253765931
2024-11-22T05:35:05,931 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732253825931
2024-11-22T05:35:05,931 INFO  [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-22T05:35:05,932 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,36973,1732253704491-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,932 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,36973,1732253704491-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,932 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,36973,1732253704491-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,932 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-82891a793a29:36973, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,932 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,932 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:05,935 DEBUG [master/82891a793a29:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-22T05:35:05,937 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.163sec
2024-11-22T05:35:05,937 INFO  [master/82891a793a29:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-22T05:35:05,937 INFO  [master/82891a793a29:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-22T05:35:05,937 INFO  [master/82891a793a29:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-22T05:35:05,937 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-22T05:35:05,937 INFO  [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-22T05:35:05,938 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,36973,1732253704491-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T05:35:05,938 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,36973,1732253704491-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-22T05:35:05,940 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-22T05:35:05,940 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-22T05:35:05,940 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,36973,1732253704491-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:06,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bb75251, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:35:06,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82891a793a29,36973,-1 for getting cluster id
2024-11-22T05:35:06,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-22T05:35:06,038 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ac4ac39-ce31-486d-9d9f-1ad34a14fa25'
2024-11-22T05:35:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-22T05:35:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ac4ac39-ce31-486d-9d9f-1ad34a14fa25"

2024-11-22T05:35:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b6e8b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:35:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82891a793a29,36973,-1]
2024-11-22T05:35:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-22T05:35:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:35:06,041 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-22T05:35:06,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@451ba750, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:35:06,043 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T05:35:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82891a793a29,35099,1732253704679, seqNum=-1]
2024-11-22T05:35:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T05:35:06,046 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T05:35:06,049 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=82891a793a29,36973,1732253704491
2024-11-22T05:35:06,049 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:06,052 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-22T05:35:06,070 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/82891a793a29:0 server-side Connection retries=45
2024-11-22T05:35:06,070 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:35:06,070 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T05:35:06,070 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T05:35:06,070 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:35:06,070 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T05:35:06,070 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-22T05:35:06,071 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T05:35:06,071 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33487
2024-11-22T05:35:06,073 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33487 connecting to ZooKeeper ensemble=127.0.0.1:59485
2024-11-22T05:35:06,074 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:06,076 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:06,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334870x0, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T05:35:06,101 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:334870x0, quorum=127.0.0.1:59485, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-11-22T05:35:06,101 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-11-22T05:35:06,101 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33487-0x1016109dcbc0002 connected
2024-11-22T05:35:06,102 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-22T05:35:06,103 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-22T05:35:06,103 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:33487-0x1016109dcbc0002, quorum=127.0.0.1:59485, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-22T05:35:06,106 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33487-0x1016109dcbc0002, quorum=127.0.0.1:59485, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T05:35:06,106 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33487
2024-11-22T05:35:06,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33487
2024-11-22T05:35:06,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33487
2024-11-22T05:35:06,113 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33487
2024-11-22T05:35:06,113 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33487
2024-11-22T05:35:06,114 INFO  [RS:1;82891a793a29:33487 {}] regionserver.HRegionServer(746): ClusterId : 8ac4ac39-ce31-486d-9d9f-1ad34a14fa25
2024-11-22T05:35:06,114 DEBUG [RS:1;82891a793a29:33487 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-22T05:35:06,121 DEBUG [RS:1;82891a793a29:33487 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-22T05:35:06,122 DEBUG [RS:1;82891a793a29:33487 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-22T05:35:06,132 DEBUG [RS:1;82891a793a29:33487 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-22T05:35:06,133 DEBUG [RS:1;82891a793a29:33487 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a95bd4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82891a793a29/172.17.0.2:0
2024-11-22T05:35:06,149 DEBUG [RS:1;82891a793a29:33487 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;82891a793a29:33487
2024-11-22T05:35:06,149 INFO  [RS:1;82891a793a29:33487 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-22T05:35:06,149 INFO  [RS:1;82891a793a29:33487 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-22T05:35:06,149 DEBUG [RS:1;82891a793a29:33487 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-22T05:35:06,150 INFO  [RS:1;82891a793a29:33487 {}] regionserver.HRegionServer(2659): reportForDuty to master=82891a793a29,36973,1732253704491 with port=33487, startcode=1732253706069
2024-11-22T05:35:06,150 DEBUG [RS:1;82891a793a29:33487 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-22T05:35:06,152 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55403, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-11-22T05:35:06,153 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36973 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 82891a793a29,33487,1732253706069
2024-11-22T05:35:06,153 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36973 {}] master.ServerManager(517): Registering regionserver=82891a793a29,33487,1732253706069
2024-11-22T05:35:06,155 DEBUG [RS:1;82891a793a29:33487 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253
2024-11-22T05:35:06,155 DEBUG [RS:1;82891a793a29:33487 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46791
2024-11-22T05:35:06,155 DEBUG [RS:1;82891a793a29:33487 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-22T05:35:06,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T05:35:06,163 DEBUG [RS:1;82891a793a29:33487 {}] zookeeper.ZKUtil(111): regionserver:33487-0x1016109dcbc0002, quorum=127.0.0.1:59485, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/82891a793a29,33487,1732253706069
2024-11-22T05:35:06,163 WARN  [RS:1;82891a793a29:33487 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T05:35:06,164 INFO  [RS:1;82891a793a29:33487 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:35:06,164 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [82891a793a29,33487,1732253706069]
2024-11-22T05:35:06,164 DEBUG [RS:1;82891a793a29:33487 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069
2024-11-22T05:35:06,168 INFO  [RS:1;82891a793a29:33487 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-22T05:35:06,171 INFO  [RS:1;82891a793a29:33487 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-22T05:35:06,171 INFO  [RS:1;82891a793a29:33487 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T05:35:06,171 INFO  [RS:1;82891a793a29:33487 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:06,171 INFO  [RS:1;82891a793a29:33487 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-22T05:35:06,173 INFO  [RS:1;82891a793a29:33487 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-22T05:35:06,173 INFO  [RS:1;82891a793a29:33487 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:06,173 DEBUG [RS:1;82891a793a29:33487 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:06,173 DEBUG [RS:1;82891a793a29:33487 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:06,173 DEBUG [RS:1;82891a793a29:33487 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:06,173 DEBUG [RS:1;82891a793a29:33487 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:06,173 DEBUG [RS:1;82891a793a29:33487 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:06,174 DEBUG [RS:1;82891a793a29:33487 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/82891a793a29:0, corePoolSize=2, maxPoolSize=2
2024-11-22T05:35:06,174 DEBUG [RS:1;82891a793a29:33487 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:06,174 DEBUG [RS:1;82891a793a29:33487 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:06,174 DEBUG [RS:1;82891a793a29:33487 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:06,174 DEBUG [RS:1;82891a793a29:33487 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:06,174 DEBUG [RS:1;82891a793a29:33487 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:06,174 DEBUG [RS:1;82891a793a29:33487 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:06,174 DEBUG [RS:1;82891a793a29:33487 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/82891a793a29:0, corePoolSize=3, maxPoolSize=3
2024-11-22T05:35:06,174 DEBUG [RS:1;82891a793a29:33487 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0, corePoolSize=3, maxPoolSize=3
2024-11-22T05:35:06,176 INFO  [RS:1;82891a793a29:33487 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:06,176 INFO  [RS:1;82891a793a29:33487 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:06,176 INFO  [RS:1;82891a793a29:33487 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:06,176 INFO  [RS:1;82891a793a29:33487 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:06,176 INFO  [RS:1;82891a793a29:33487 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:06,176 INFO  [RS:1;82891a793a29:33487 {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,33487,1732253706069-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T05:35:06,193 INFO  [RS:1;82891a793a29:33487 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-22T05:35:06,193 INFO  [RS:1;82891a793a29:33487 {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,33487,1732253706069-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:06,193 INFO  [RS:1;82891a793a29:33487 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:06,193 INFO  [RS:1;82891a793a29:33487 {}] regionserver.Replication(171): 82891a793a29,33487,1732253706069 started
2024-11-22T05:35:06,213 INFO  [RS:1;82891a793a29:33487 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:06,213 INFO  [RS:1;82891a793a29:33487 {}] regionserver.HRegionServer(1482): Serving as 82891a793a29,33487,1732253706069, RpcServer on 82891a793a29/172.17.0.2:33487, sessionid=0x1016109dcbc0002
2024-11-22T05:35:06,213 DEBUG [RS:1;82891a793a29:33487 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-22T05:35:06,213 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;82891a793a29:33487,5,FailOnTimeoutGroup]
2024-11-22T05:35:06,213 DEBUG [RS:1;82891a793a29:33487 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 82891a793a29,33487,1732253706069
2024-11-22T05:35:06,213 DEBUG [RS:1;82891a793a29:33487 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82891a793a29,33487,1732253706069'
2024-11-22T05:35:06,213 DEBUG [RS:1;82891a793a29:33487 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-22T05:35:06,214 DEBUG [RS:1;82891a793a29:33487 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-22T05:35:06,215 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-11-22T05:35:06,215 DEBUG [RS:1;82891a793a29:33487 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-22T05:35:06,215 DEBUG [RS:1;82891a793a29:33487 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-22T05:35:06,215 DEBUG [RS:1;82891a793a29:33487 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 82891a793a29,33487,1732253706069
2024-11-22T05:35:06,215 DEBUG [RS:1;82891a793a29:33487 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82891a793a29,33487,1732253706069'
2024-11-22T05:35:06,215 DEBUG [RS:1;82891a793a29:33487 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-22T05:35:06,215 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-22T05:35:06,215 DEBUG [RS:1;82891a793a29:33487 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-22T05:35:06,216 DEBUG [RS:1;82891a793a29:33487 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-22T05:35:06,216 INFO  [RS:1;82891a793a29:33487 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-22T05:35:06,216 INFO  [RS:1;82891a793a29:33487 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-22T05:35:06,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is 82891a793a29,36973,1732253704491
2024-11-22T05:35:06,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@13d4e9ab
2024-11-22T05:35:06,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-22T05:35:06,219 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-22T05:35:06,220 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36973 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-22T05:35:06,220 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36973 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-22T05:35:06,220 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36973 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T05:35:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36973 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-11-22T05:35:06,223 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-11-22T05:35:06,224 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:06,224 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36973 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-11-22T05:35:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36973 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T05:35:06,225 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-22T05:35:06,238 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41019 is added to blk_1073741835_1011 (size=393)
2024-11-22T05:35:06,238 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741835_1011 (size=393)
2024-11-22T05:35:06,241 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 767b8467d7ba6cb07f05436770f142ec, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253
2024-11-22T05:35:06,248 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741836_1012 (size=76)
2024-11-22T05:35:06,249 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41019 is added to blk_1073741836_1012 (size=76)
2024-11-22T05:35:06,249 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:35:06,249 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 767b8467d7ba6cb07f05436770f142ec, disabling compactions & flushes
2024-11-22T05:35:06,249 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.
2024-11-22T05:35:06,249 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.
2024-11-22T05:35:06,249 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec. after waiting 0 ms
2024-11-22T05:35:06,249 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.
2024-11-22T05:35:06,249 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.
2024-11-22T05:35:06,249 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 767b8467d7ba6cb07f05436770f142ec:
Waiting for close lock at 1732253706249Disabling compacts and flushes for region at 1732253706249Disabling writes for close at 1732253706249Writing region close event to WAL at 1732253706249Closed at 1732253706249
2024-11-22T05:35:06,251 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-11-22T05:35:06,251 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1732253706251"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732253706251"}]},"ts":"1732253706251"}
2024-11-22T05:35:06,254 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-22T05:35:06,255 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-22T05:35:06,256 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732253706255"}]},"ts":"1732253706255"}
2024-11-22T05:35:06,258 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-11-22T05:35:06,259 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=767b8467d7ba6cb07f05436770f142ec, ASSIGN}]
2024-11-22T05:35:06,261 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=767b8467d7ba6cb07f05436770f142ec, ASSIGN
2024-11-22T05:35:06,262 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=767b8467d7ba6cb07f05436770f142ec, ASSIGN; state=OFFLINE, location=82891a793a29,35099,1732253704679; forceNewPlan=false, retain=false
2024-11-22T05:35:06,318 INFO  [RS:1;82891a793a29:33487 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C33487%2C1732253706069, suffix=, logDir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069, archiveDir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/oldWALs, maxLogs=32
2024-11-22T05:35:06,319 INFO  [RS:1;82891a793a29:33487 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C33487%2C1732253706069.1732253706319
2024-11-22T05:35:06,326 INFO  [RS:1;82891a793a29:33487 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
2024-11-22T05:35:06,326 DEBUG [RS:1;82891a793a29:33487 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34441:34441),(127.0.0.1/127.0.0.1:37311:37311)]
2024-11-22T05:35:06,414 INFO  [82891a793a29:36973 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-11-22T05:35:06,414 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=767b8467d7ba6cb07f05436770f142ec, regionState=OPENING, regionLocation=82891a793a29,35099,1732253704679
2024-11-22T05:35:06,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=767b8467d7ba6cb07f05436770f142ec, ASSIGN because future has completed
2024-11-22T05:35:06,420 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 767b8467d7ba6cb07f05436770f142ec, server=82891a793a29,35099,1732253704679}]
2024-11-22T05:35:06,581 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.
2024-11-22T05:35:06,582 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 767b8467d7ba6cb07f05436770f142ec, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:35:06,583 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:06,583 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:35:06,583 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:06,583 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:06,586 INFO  [StoreOpener-767b8467d7ba6cb07f05436770f142ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 767b8467d7ba6cb07f05436770f142ec 
2024-11-22T05:35:06,587 INFO  [StoreOpener-767b8467d7ba6cb07f05436770f142ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 767b8467d7ba6cb07f05436770f142ec columnFamilyName info
2024-11-22T05:35:06,587 DEBUG [StoreOpener-767b8467d7ba6cb07f05436770f142ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:06,588 INFO  [StoreOpener-767b8467d7ba6cb07f05436770f142ec-1 {}] regionserver.HStore(327): Store=767b8467d7ba6cb07f05436770f142ec/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:35:06,588 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:06,589 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:06,589 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:06,590 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:06,590 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:06,592 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:06,594 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:35:06,595 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 767b8467d7ba6cb07f05436770f142ec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=763186, jitterRate=-0.029559656977653503}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-22T05:35:06,595 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:06,596 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 767b8467d7ba6cb07f05436770f142ec:
Running coprocessor pre-open hook at 1732253706583Writing region info on filesystem at 1732253706583Initializing all the Stores at 1732253706585 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253706585Cleaning up temporary data from old regions at 1732253706590 (+5 ms)Running coprocessor post-open hooks at 1732253706595 (+5 ms)Region opened successfully at 1732253706596 (+1 ms)
2024-11-22T05:35:06,597 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec., pid=6, masterSystemTime=1732253706575
2024-11-22T05:35:06,600 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.
2024-11-22T05:35:06,601 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.
2024-11-22T05:35:06,602 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=767b8467d7ba6cb07f05436770f142ec, regionState=OPEN, openSeqNum=2, regionLocation=82891a793a29,35099,1732253704679
2024-11-22T05:35:06,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 767b8467d7ba6cb07f05436770f142ec, server=82891a793a29,35099,1732253704679 because future has completed
2024-11-22T05:35:06,611 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-22T05:35:06,611 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 767b8467d7ba6cb07f05436770f142ec, server=82891a793a29,35099,1732253704679 in 187 msec
2024-11-22T05:35:06,615 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-22T05:35:06,615 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=767b8467d7ba6cb07f05436770f142ec, ASSIGN in 352 msec
2024-11-22T05:35:06,616 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-22T05:35:06,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732253706616"}]},"ts":"1732253706616"}
2024-11-22T05:35:06,619 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-11-22T05:35:06,621 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-11-22T05:35:06,623 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 401 msec
2024-11-22T05:35:09,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-22T05:35:09,491 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-22T05:35:09,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-22T05:35:09,492 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-11-22T05:35:09,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T05:35:09,493 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-22T05:35:11,353 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T05:35:11,356 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:11,381 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:11,383 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:11,384 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:11,399 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-11-22T05:35:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36973 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T05:35:16,284 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-11-22T05:35:16,284 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-11-22T05:35:16,287 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-11-22T05:35:16,287 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.
2024-11-22T05:35:16,302 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:35:16,306 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:35:16,306 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:35:16,306 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:35:16,306 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:35:16,307 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cac6b83{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:35:16,307 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5dc4ef73{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:35:16,409 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@65429201{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/java.io.tmpdir/jetty-localhost-33799-hadoop-hdfs-3_4_1-tests_jar-_-any-8532085003016441862/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:35:16,410 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4cbb1003{HTTP/1.1, (http/1.1)}{localhost:33799}
2024-11-22T05:35:16,410 INFO  [Time-limited test {}] server.Server(415): Started @123667ms
2024-11-22T05:35:16,412 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:35:16,448 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:35:16,451 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:35:16,452 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:35:16,452 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:35:16,453 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:35:16,453 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3eb2cdd3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:35:16,454 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5bea65f7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:35:16,560 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@70a0068f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/java.io.tmpdir/jetty-localhost-42895-hadoop-hdfs-3_4_1-tests_jar-_-any-12314336395692755121/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:35:16,561 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@39615ad8{HTTP/1.1, (http/1.1)}{localhost:42895}
2024-11-22T05:35:16,561 INFO  [Time-limited test {}] server.Server(415): Started @123818ms
2024-11-22T05:35:16,562 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:35:16,607 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:35:16,611 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:35:16,611 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:35:16,612 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:35:16,612 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:35:16,612 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264a677b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:35:16,612 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@158a9d8c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:35:16,719 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5ec5aa68{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/java.io.tmpdir/jetty-localhost-38807-hadoop-hdfs-3_4_1-tests_jar-_-any-8456615318688635304/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:35:16,719 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@393eb51f{HTTP/1.1, (http/1.1)}{localhost:38807}
2024-11-22T05:35:16,719 INFO  [Time-limited test {}] server.Server(415): Started @123977ms
2024-11-22T05:35:16,721 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:35:17,785 WARN  [Thread-868 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data6/current/BP-860069939-172.17.0.2-1732253702022/current, will proceed with Du for space computation calculation, 
2024-11-22T05:35:17,785 WARN  [Thread-867 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data5/current/BP-860069939-172.17.0.2-1732253702022/current, will proceed with Du for space computation calculation, 
2024-11-22T05:35:17,812 WARN  [Thread-809 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:35:17,815 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7e9f466554466461 with lease ID 0x2f19ab655b3c7ebe: Processing first storage report for DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b from datanode DatanodeRegistration(127.0.0.1:44683, datanodeUuid=f064e889-3229-4dfa-a3db-ce683bec0b02, infoPort=33825, infoSecurePort=0, ipcPort=33751, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022)
2024-11-22T05:35:17,815 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7e9f466554466461 with lease ID 0x2f19ab655b3c7ebe: from storage DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b node DatanodeRegistration(127.0.0.1:44683, datanodeUuid=f064e889-3229-4dfa-a3db-ce683bec0b02, infoPort=33825, infoSecurePort=0, ipcPort=33751, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:35:17,815 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7e9f466554466461 with lease ID 0x2f19ab655b3c7ebe: Processing first storage report for DS-daa89a5d-4861-452a-a2f5-775958329d8b from datanode DatanodeRegistration(127.0.0.1:44683, datanodeUuid=f064e889-3229-4dfa-a3db-ce683bec0b02, infoPort=33825, infoSecurePort=0, ipcPort=33751, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022)
2024-11-22T05:35:17,815 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7e9f466554466461 with lease ID 0x2f19ab655b3c7ebe: from storage DS-daa89a5d-4861-452a-a2f5-775958329d8b node DatanodeRegistration(127.0.0.1:44683, datanodeUuid=f064e889-3229-4dfa-a3db-ce683bec0b02, infoPort=33825, infoSecurePort=0, ipcPort=33751, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:35:18,251 WARN  [Thread-879 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7/current/BP-860069939-172.17.0.2-1732253702022/current, will proceed with Du for space computation calculation, 
2024-11-22T05:35:18,251 WARN  [Thread-880 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8/current/BP-860069939-172.17.0.2-1732253702022/current, will proceed with Du for space computation calculation, 
2024-11-22T05:35:18,276 WARN  [Thread-831 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:35:18,278 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x808dd868564d6ed8 with lease ID 0x2f19ab655b3c7ebf: Processing first storage report for DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01 from datanode DatanodeRegistration(127.0.0.1:36049, datanodeUuid=997782d0-6ecf-48b6-aea7-d2c60565a4a7, infoPort=33343, infoSecurePort=0, ipcPort=45673, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022)
2024-11-22T05:35:18,278 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x808dd868564d6ed8 with lease ID 0x2f19ab655b3c7ebf: from storage DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01 node DatanodeRegistration(127.0.0.1:36049, datanodeUuid=997782d0-6ecf-48b6-aea7-d2c60565a4a7, infoPort=33343, infoSecurePort=0, ipcPort=45673, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:35:18,278 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x808dd868564d6ed8 with lease ID 0x2f19ab655b3c7ebf: Processing first storage report for DS-0e62748a-3c1d-406d-86ae-e2615c47cd35 from datanode DatanodeRegistration(127.0.0.1:36049, datanodeUuid=997782d0-6ecf-48b6-aea7-d2c60565a4a7, infoPort=33343, infoSecurePort=0, ipcPort=45673, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022)
2024-11-22T05:35:18,279 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x808dd868564d6ed8 with lease ID 0x2f19ab655b3c7ebf: from storage DS-0e62748a-3c1d-406d-86ae-e2615c47cd35 node DatanodeRegistration(127.0.0.1:36049, datanodeUuid=997782d0-6ecf-48b6-aea7-d2c60565a4a7, infoPort=33343, infoSecurePort=0, ipcPort=45673, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:35:18,354 WARN  [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data10/current/BP-860069939-172.17.0.2-1732253702022/current, will proceed with Du for space computation calculation, 
2024-11-22T05:35:18,353 WARN  [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data9/current/BP-860069939-172.17.0.2-1732253702022/current, will proceed with Du for space computation calculation, 
2024-11-22T05:35:18,375 WARN  [Thread-853 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:35:18,378 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3ad0be8a77fc1605 with lease ID 0x2f19ab655b3c7ec0: Processing first storage report for DS-181c5212-0c5a-4745-b234-bc5e6713eebb from datanode DatanodeRegistration(127.0.0.1:44489, datanodeUuid=ae3bd80a-f23b-4275-95b4-0e2b76378b1d, infoPort=36257, infoSecurePort=0, ipcPort=40451, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022)
2024-11-22T05:35:18,378 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3ad0be8a77fc1605 with lease ID 0x2f19ab655b3c7ec0: from storage DS-181c5212-0c5a-4745-b234-bc5e6713eebb node DatanodeRegistration(127.0.0.1:44489, datanodeUuid=ae3bd80a-f23b-4275-95b4-0e2b76378b1d, infoPort=36257, infoSecurePort=0, ipcPort=40451, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:35:18,378 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3ad0be8a77fc1605 with lease ID 0x2f19ab655b3c7ec0: Processing first storage report for DS-0d9a41d1-6861-4b39-a041-18d580a3a91f from datanode DatanodeRegistration(127.0.0.1:44489, datanodeUuid=ae3bd80a-f23b-4275-95b4-0e2b76378b1d, infoPort=36257, infoSecurePort=0, ipcPort=40451, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022)
2024-11-22T05:35:18,378 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3ad0be8a77fc1605 with lease ID 0x2f19ab655b3c7ec0: from storage DS-0d9a41d1-6861-4b39-a041-18d580a3a91f node DatanodeRegistration(127.0.0.1:44489, datanodeUuid=ae3bd80a-f23b-4275-95b4-0e2b76378b1d, infoPort=36257, infoSecurePort=0, ipcPort=40451, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:35:18,457 WARN  [ResponseProcessor for block BP-860069939-172.17.0.2-1732253702022:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-860069939-172.17.0.2-1732253702022:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:18,457 WARN  [ResponseProcessor for block BP-860069939-172.17.0.2-1732253702022:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-860069939-172.17.0.2-1732253702022:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:18,458 WARN  [DataStreamer for file /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta block BP-860069939-172.17.0.2-1732253702022:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK], DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]) is bad.
2024-11-22T05:35:18,458 WARN  [DataStreamer for file /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/WALs/82891a793a29,36973,1732253704491/82891a793a29%2C36973%2C1732253704491.1732253704853 block BP-860069939-172.17.0.2-1732253702022:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK], DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]) is bad.
2024-11-22T05:35:18,459 WARN  [ResponseProcessor for block BP-860069939-172.17.0.2-1732253702022:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-860069939-172.17.0.2-1732253702022:blk_1073741837_1013
java.io.IOException: Bad response ERROR for BP-860069939-172.17.0.2-1732253702022:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:18,459 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:40162 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41959:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40162 dst: /127.0.0.1:41959
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:18,459 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-358693681_22 at /127.0.0.1:40116 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41959:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40116 dst: /127.0.0.1:41959
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:18,460 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-358693681_22 at /127.0.0.1:42950 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41019:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42950 dst: /127.0.0.1:41019
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:18,460 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:43000 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41019:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43000 dst: /127.0.0.1:41019
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:18,461 WARN  [ResponseProcessor for block BP-860069939-172.17.0.2-1732253702022:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-860069939-172.17.0.2-1732253702022:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-860069939-172.17.0.2-1732253702022:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:18,462 WARN  [DataStreamer for file /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319 block BP-860069939-172.17.0.2-1732253702022:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK], DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]) is bad.
2024-11-22T05:35:18,462 WARN  [DataStreamer for file /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253705361 block BP-860069939-172.17.0.2-1732253702022:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK], DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]) is bad.
2024-11-22T05:35:18,462 WARN  [PacketResponder: BP-860069939-172.17.0.2-1732253702022:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41959] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:18,463 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3efce601{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:35:18,463 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:42986 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41019:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42986 dst: /127.0.0.1:41019
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:18,463 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:40146 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41959:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40146 dst: /127.0.0.1:41959
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:18,463 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@e79a3d4{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:35:18,463 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:35:18,464 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2c64d82b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:35:18,464 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@217a95d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.log.dir/,STOPPED}
2024-11-22T05:35:18,462 WARN  [PacketResponder: BP-860069939-172.17.0.2-1732253702022:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41959] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:18,464 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1006718637_22 at /127.0.0.1:43036 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:41019:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43036 dst: /127.0.0.1:41019
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:18,465 WARN  [BP-860069939-172.17.0.2-1732253702022 heartbeating to localhost/127.0.0.1:46791 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:35:18,465 WARN  [BP-860069939-172.17.0.2-1732253702022 heartbeating to localhost/127.0.0.1:46791 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-860069939-172.17.0.2-1732253702022 (Datanode Uuid 7022ae67-a4d8-42d1-b7a4-82fbd184c97e) service to localhost/127.0.0.1:46791
2024-11-22T05:35:18,465 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:35:18,465 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:35:18,465 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1006718637_22 at /127.0.0.1:40202 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:41959:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40202 dst: /127.0.0.1:41959
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 47860 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:18,466 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data3/current/BP-860069939-172.17.0.2-1732253702022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:35:18,466 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data4/current/BP-860069939-172.17.0.2-1732253702022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:35:18,466 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:35:18,467 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@7e9aa006 {}] datanode.DataXceiver(331): 127.0.0.1:41019:DataXceiver error processing unknown operation  src: /127.0.0.1:57820 dst: /127.0.0.1:41019
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:18,467 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@6e4ede9d {}] datanode.DataXceiver(331): 127.0.0.1:41019:DataXceiver error processing unknown operation  src: /127.0.0.1:57818 dst: /127.0.0.1:41019
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:18,467 WARN  [DataStreamer for file /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta block BP-860069939-172.17.0.2-1732253702022:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:18,468 WARN  [DataStreamer for file /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319 block BP-860069939-172.17.0.2-1732253702022:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:18,468 WARN  [DataStreamer for file /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253705361 block BP-860069939-172.17.0.2-1732253702022:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:18,468 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@3a0d7c36 {}] datanode.DataXceiver(331): 127.0.0.1:41019:DataXceiver error processing unknown operation  src: /127.0.0.1:57816 dst: /127.0.0.1:41019
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:18,469 WARN  [DataStreamer for file /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/WALs/82891a793a29,36973,1732253704491/82891a793a29%2C36973%2C1732253704491.1732253704853 block BP-860069939-172.17.0.2-1732253702022:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:18,470 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bba803f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:35:18,470 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7629a449{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:35:18,470 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:35:18,470 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7a18c5e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:35:18,471 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@372d60ec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.log.dir/,STOPPED}
2024-11-22T05:35:18,472 WARN  [BP-860069939-172.17.0.2-1732253702022 heartbeating to localhost/127.0.0.1:46791 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:35:18,472 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:35:18,472 WARN  [BP-860069939-172.17.0.2-1732253702022 heartbeating to localhost/127.0.0.1:46791 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-860069939-172.17.0.2-1732253702022 (Datanode Uuid 4b522218-4cbe-42f6-948e-e6c30b984188) service to localhost/127.0.0.1:46791
2024-11-22T05:35:18,472 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:35:18,472 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data1/current/BP-860069939-172.17.0.2-1732253702022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:35:18,473 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data2/current/BP-860069939-172.17.0.2-1732253702022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:35:18,473 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:35:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec., hostname=82891a793a29,35099,1732253704679, seqNum=2]
2024-11-22T05:35:18,478 ERROR [FSHLog-0-hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253-prefix:82891a793a29,35099,1732253704679 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:18,478 WARN  [FSHLog-0-hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253-prefix:82891a793a29,35099,1732253704679 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:18,478 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:18,479 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 82891a793a29%2C35099%2C1732253704679:(num 1732253705361) roll requested
2024-11-22T05:35:18,479 INFO  [regionserver/82891a793a29:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C35099%2C1732253704679.1732253718479
2024-11-22T05:35:18,485 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:18,485 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:18,485 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:18,485 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:18,485 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:18,486 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253705361 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253718479
2024-11-22T05:35:18,486 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:18,486 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:18,487 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-11-22T05:35:18,488 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-11-22T05:35:18,488 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253705361
2024-11-22T05:35:18,490 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36257:36257),(127.0.0.1/127.0.0.1:33825:33825)]
2024-11-22T05:35:18,490 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253705361 is not closed yet, will try archiving it next time
2024-11-22T05:35:18,491 WARN  [IPC Server handler 1 on default port 46791 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253705361 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1009
2024-11-22T05:35:18,495 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253705361 after 5ms
2024-11-22T05:35:18,827 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:20,176 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:20,490 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:20,492 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253718479
2024-11-22T05:35:20,493 WARN  [ResponseProcessor for block BP-860069939-172.17.0.2-1732253702022:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-860069939-172.17.0.2-1732253702022:blk_1073741838_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:20,493 WARN  [DataStreamer for file /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253718479 block BP-860069939-172.17.0.2-1732253702022:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK], DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]) is bad.
2024-11-22T05:35:20,495 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:41416 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:44489:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41416 dst: /127.0.0.1:44489
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:20,496 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:43320 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:44683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43320 dst: /127.0.0.1:44683
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:20,497 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5ec5aa68{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:35:20,498 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@393eb51f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:35:20,498 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:35:20,499 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@158a9d8c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:35:20,499 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264a677b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.log.dir/,STOPPED}
2024-11-22T05:35:20,501 WARN  [BP-860069939-172.17.0.2-1732253702022 heartbeating to localhost/127.0.0.1:46791 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:35:20,501 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:35:20,501 WARN  [BP-860069939-172.17.0.2-1732253702022 heartbeating to localhost/127.0.0.1:46791 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-860069939-172.17.0.2-1732253702022 (Datanode Uuid ae3bd80a-f23b-4275-95b4-0e2b76378b1d) service to localhost/127.0.0.1:46791
2024-11-22T05:35:20,501 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:35:20,502 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data9/current/BP-860069939-172.17.0.2-1732253702022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:35:20,502 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data10/current/BP-860069939-172.17.0.2-1732253702022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:35:20,502 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:35:20,827 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:22,176 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:22,491 WARN  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]]
2024-11-22T05:35:22,491 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:22,491 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 82891a793a29%2C35099%2C1732253704679:(num 1732253718479) roll requested
2024-11-22T05:35:22,492 INFO  [regionserver/82891a793a29:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C35099%2C1732253704679.1732253722492
2024-11-22T05:35:22,495 WARN  [Thread-913 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1021
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:22,495 WARN  [Thread-913 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK], DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]) is bad.
2024-11-22T05:35:22,495 WARN  [Thread-913 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741839_1021
2024-11-22T05:35:22,498 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253705361 after 4010ms
2024-11-22T05:35:22,501 WARN  [Thread-913 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]
2024-11-22T05:35:22,505 WARN  [Thread-913 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:22,506 WARN  [Thread-913 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK], DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]) is bad.
2024-11-22T05:35:22,506 WARN  [Thread-913 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741840_1022
2024-11-22T05:35:22,507 WARN  [Thread-913 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]
2024-11-22T05:35:22,507 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-22T05:35:22,515 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:22,515 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:22,515 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:22,515 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:22,515 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:22,516 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253718479 with entries=3, filesize=3.51 KB; new WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253722492
2024-11-22T05:35:22,518 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44683 is added to blk_1073741838_1020 (size=3600)
2024-11-22T05:35:22,519 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253705361 is not closed yet, will try archiving it next time
2024-11-22T05:35:22,528 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:33343:33343)]
2024-11-22T05:35:22,528 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253705361 is not closed yet, will try archiving it next time
2024-11-22T05:35:22,828 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:23,830 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741838_1020 (size=3600)
2024-11-22T05:35:24,176 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:24,515 WARN  [ResponseProcessor for block BP-860069939-172.17.0.2-1732253702022:blk_1073741841_1023 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-860069939-172.17.0.2-1732253702022:blk_1073741841_1023
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:24,515 WARN  [DataStreamer for file /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253722492 block BP-860069939-172.17.0.2-1732253702022:blk_1073741841_1023 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK], DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]) is bad.
2024-11-22T05:35:24,516 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:56662 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:44683:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56662 dst: /127.0.0.1:44683
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:24,516 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45538 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:36049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45538 dst: /127.0.0.1:36049
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:24,529 WARN  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK]]
2024-11-22T05:35:24,529 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:24,529 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 82891a793a29%2C35099%2C1732253704679:(num 1732253722492) roll requested
2024-11-22T05:35:24,530 INFO  [regionserver/82891a793a29:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C35099%2C1732253704679.1732253724529
2024-11-22T05:35:24,535 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41959
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:24,534 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45570 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741842_1025] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8]'}, localName='127.0.0.1:36049', datanodeUuid='997782d0-6ecf-48b6-aea7-d2c60565a4a7', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741842_1025 to mirror 127.0.0.1:41959
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:24,535 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK], DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]) is bad.
2024-11-22T05:35:24,535 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741842_1025
2024-11-22T05:35:24,535 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45570 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741842_1025] {}] datanode.BlockReceiver(316): Block 1073741842 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-22T05:35:24,535 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45570 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741842_1025] {}] datanode.DataXceiver(331): 127.0.0.1:36049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45570 dst: /127.0.0.1:36049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:24,536 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]
2024-11-22T05:35:24,538 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:24,539 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK], DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]) is bad.
2024-11-22T05:35:24,539 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741843_1026
2024-11-22T05:35:24,540 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]
2024-11-22T05:35:24,543 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41019
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:24,543 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK], DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]) is bad.
2024-11-22T05:35:24,543 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45580 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741844_1027] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8]'}, localName='127.0.0.1:36049', datanodeUuid='997782d0-6ecf-48b6-aea7-d2c60565a4a7', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741844_1027 to mirror 127.0.0.1:41019
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:24,543 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741844_1027
2024-11-22T05:35:24,544 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45580 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741844_1027] {}] datanode.BlockReceiver(316): Block 1073741844 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-22T05:35:24,544 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@65429201{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:35:24,544 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45580 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741844_1027] {}] datanode.DataXceiver(331): 127.0.0.1:36049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45580 dst: /127.0.0.1:36049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:24,544 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]
2024-11-22T05:35:24,544 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4cbb1003{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:35:24,544 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:35:24,544 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5dc4ef73{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:35:24,544 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cac6b83{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.log.dir/,STOPPED}
2024-11-22T05:35:24,545 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:24,546 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK], DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]) is bad.
2024-11-22T05:35:24,546 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741845_1028
2024-11-22T05:35:24,547 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]
2024-11-22T05:35:24,547 WARN  [BP-860069939-172.17.0.2-1732253702022 heartbeating to localhost/127.0.0.1:46791 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:35:24,547 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:35:24,547 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:35:24,547 WARN  [BP-860069939-172.17.0.2-1732253702022 heartbeating to localhost/127.0.0.1:46791 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-860069939-172.17.0.2-1732253702022 (Datanode Uuid f064e889-3229-4dfa-a3db-ce683bec0b02) service to localhost/127.0.0.1:46791
2024-11-22T05:35:24,547 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data5/current/BP-860069939-172.17.0.2-1732253702022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:35:24,547 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data6/current/BP-860069939-172.17.0.2-1732253702022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:35:24,547 WARN  [IPC Server handler 4 on default port 46791 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-22T05:35:24,548 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:35:24,548 WARN  [IPC Server handler 4 on default port 46791 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-22T05:35:24,548 WARN  [IPC Server handler 4 on default port 46791 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-22T05:35:24,551 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:24,551 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:24,551 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:24,551 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:24,551 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:24,551 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253722492 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253724529
2024-11-22T05:35:24,552 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33343:33343)]
2024-11-22T05:35:24,552 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253705361 is not closed yet, will try archiving it next time
2024-11-22T05:35:24,553 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253722492 is not closed yet, will try archiving it next time
2024-11-22T05:35:24,553 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741841_1024 (size=1262)
2024-11-22T05:35:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35099 {}] regionserver.HRegion(8855): Flush requested on 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:24,558 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 767b8467d7ba6cb07f05436770f142ec 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T05:35:24,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/d704b6acfbc94fa599eefbdb4da3281d is 1080, key is row0002/info:/1732253720503/Put/seqid=0
2024-11-22T05:35:24,576 WARN  [Thread-930 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:24,576 WARN  [Thread-930 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK], DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]) is bad.
2024-11-22T05:35:24,576 WARN  [Thread-930 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741847_1030
2024-11-22T05:35:24,576 WARN  [Thread-930 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]
2024-11-22T05:35:24,578 WARN  [Thread-930 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44489
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:24,578 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45606 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741848_1031] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8]'}, localName='127.0.0.1:36049', datanodeUuid='997782d0-6ecf-48b6-aea7-d2c60565a4a7', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741848_1031 to mirror 127.0.0.1:44489
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:24,579 WARN  [Thread-930 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK], DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]) is bad.
2024-11-22T05:35:24,579 WARN  [Thread-930 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741848_1031
2024-11-22T05:35:24,579 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45606 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741848_1031] {}] datanode.BlockReceiver(316): Block 1073741848 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T05:35:24,579 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45606 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741848_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45606 dst: /127.0.0.1:36049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:24,579 WARN  [Thread-930 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]
2024-11-22T05:35:24,580 WARN  [Thread-930 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:24,581 WARN  [Thread-930 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK], DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]) is bad.
2024-11-22T05:35:24,581 WARN  [Thread-930 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741849_1032
2024-11-22T05:35:24,581 WARN  [Thread-930 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]
2024-11-22T05:35:24,582 WARN  [Thread-930 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:24,583 WARN  [Thread-930 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK], DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]) is bad.
2024-11-22T05:35:24,583 WARN  [Thread-930 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741850_1033
2024-11-22T05:35:24,583 WARN  [Thread-930 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]
2024-11-22T05:35:24,584 WARN  [IPC Server handler 3 on default port 46791 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-22T05:35:24,584 WARN  [IPC Server handler 3 on default port 46791 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-22T05:35:24,584 WARN  [IPC Server handler 3 on default port 46791 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-22T05:35:24,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741851_1034 (size=10347)
2024-11-22T05:35:24,828 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:24,955 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253705361 is not closed yet, will try archiving it next time
2024-11-22T05:35:24,988 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/d704b6acfbc94fa599eefbdb4da3281d
2024-11-22T05:35:24,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/d704b6acfbc94fa599eefbdb4da3281d as hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/d704b6acfbc94fa599eefbdb4da3281d
2024-11-22T05:35:25,006 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/d704b6acfbc94fa599eefbdb4da3281d, entries=5, sequenceid=11, filesize=10.1 K
2024-11-22T05:35:25,008 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 767b8467d7ba6cb07f05436770f142ec in 451ms, sequenceid=11, compaction requested=false
2024-11-22T05:35:25,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 767b8467d7ba6cb07f05436770f142ec:

2024-11-22T05:35:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35099 {}] regionserver.HRegion(8855): Flush requested on 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:25,182 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 767b8467d7ba6cb07f05436770f142ec 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-11-22T05:35:25,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/b6595af9348041e4a438b01eb70d0b43 is 1080, key is row0007/info:/1732253724558/Put/seqid=0
2024-11-22T05:35:25,191 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:25,191 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK], DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]) is bad.
2024-11-22T05:35:25,191 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741852_1035
2024-11-22T05:35:25,192 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]
2024-11-22T05:35:25,194 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41019
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:25,194 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45626 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741853_1036] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8]'}, localName='127.0.0.1:36049', datanodeUuid='997782d0-6ecf-48b6-aea7-d2c60565a4a7', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741853_1036 to mirror 127.0.0.1:41019
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:25,195 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK], DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]) is bad.
2024-11-22T05:35:25,195 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45626 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741853_1036] {}] datanode.BlockReceiver(316): Block 1073741853 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T05:35:25,195 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741853_1036
2024-11-22T05:35:25,195 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45626 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741853_1036] {}] datanode.DataXceiver(331): 127.0.0.1:36049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45626 dst: /127.0.0.1:36049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:25,195 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]
2024-11-22T05:35:25,198 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41959
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:25,198 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45632 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741854_1037] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8]'}, localName='127.0.0.1:36049', datanodeUuid='997782d0-6ecf-48b6-aea7-d2c60565a4a7', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741854_1037 to mirror 127.0.0.1:41959
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:25,199 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK], DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]) is bad.
2024-11-22T05:35:25,199 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741854_1037
2024-11-22T05:35:25,199 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45632 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741854_1037] {}] datanode.BlockReceiver(316): Block 1073741854 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T05:35:25,199 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45632 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741854_1037] {}] datanode.DataXceiver(331): 127.0.0.1:36049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45632 dst: /127.0.0.1:36049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:25,199 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]
2024-11-22T05:35:25,201 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:25,201 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK], DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]) is bad.
2024-11-22T05:35:25,201 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741855_1038
2024-11-22T05:35:25,202 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]
2024-11-22T05:35:25,203 WARN  [IPC Server handler 2 on default port 46791 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-22T05:35:25,203 WARN  [IPC Server handler 2 on default port 46791 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-22T05:35:25,203 WARN  [IPC Server handler 2 on default port 46791 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-22T05:35:25,206 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741856_1039 (size=12506)
2024-11-22T05:35:25,607 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/b6595af9348041e4a438b01eb70d0b43
2024-11-22T05:35:25,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/b6595af9348041e4a438b01eb70d0b43 as hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/b6595af9348041e4a438b01eb70d0b43
2024-11-22T05:35:25,623 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/b6595af9348041e4a438b01eb70d0b43, entries=7, sequenceid=24, filesize=12.2 K
2024-11-22T05:35:25,625 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for 767b8467d7ba6cb07f05436770f142ec in 442ms, sequenceid=24, compaction requested=false
2024-11-22T05:35:25,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 767b8467d7ba6cb07f05436770f142ec:

2024-11-22T05:35:25,625 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-11-22T05:35:25,625 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:35:25,625 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/b6595af9348041e4a438b01eb70d0b43 because midkey is the same as first or last row
2024-11-22T05:35:26,177 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:26,553 WARN  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK]]
2024-11-22T05:35:26,553 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:26,553 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 82891a793a29%2C35099%2C1732253704679:(num 1732253724529) roll requested
2024-11-22T05:35:26,554 INFO  [regionserver/82891a793a29:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C35099%2C1732253704679.1732253726553
2024-11-22T05:35:26,559 WARN  [Thread-942 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44489
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:26,559 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45652 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741857_1040] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8]'}, localName='127.0.0.1:36049', datanodeUuid='997782d0-6ecf-48b6-aea7-d2c60565a4a7', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741857_1040 to mirror 127.0.0.1:44489
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:26,559 WARN  [Thread-942 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK], DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]) is bad.
2024-11-22T05:35:26,560 WARN  [Thread-942 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741857_1040
2024-11-22T05:35:26,560 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45652 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741857_1040] {}] datanode.BlockReceiver(316): Block 1073741857 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-22T05:35:26,560 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45652 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741857_1040] {}] datanode.DataXceiver(331): 127.0.0.1:36049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45652 dst: /127.0.0.1:36049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:26,560 WARN  [Thread-942 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]
2024-11-22T05:35:26,562 WARN  [Thread-942 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:26,562 WARN  [Thread-942 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK], DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]) is bad.
2024-11-22T05:35:26,562 WARN  [Thread-942 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741858_1041
2024-11-22T05:35:26,563 WARN  [Thread-942 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]
2024-11-22T05:35:26,565 WARN  [Thread-942 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:26,565 WARN  [Thread-942 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK], DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]) is bad.
2024-11-22T05:35:26,565 WARN  [Thread-942 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741859_1042
2024-11-22T05:35:26,566 WARN  [Thread-942 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]
2024-11-22T05:35:26,568 WARN  [Thread-942 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41959
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:26,568 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45664 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741860_1043] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8]'}, localName='127.0.0.1:36049', datanodeUuid='997782d0-6ecf-48b6-aea7-d2c60565a4a7', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741860_1043 to mirror 127.0.0.1:41959
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:26,569 WARN  [Thread-942 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK], DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]) is bad.
2024-11-22T05:35:26,569 WARN  [Thread-942 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741860_1043
2024-11-22T05:35:26,569 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45664 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741860_1043] {}] datanode.BlockReceiver(316): Block 1073741860 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-22T05:35:26,569 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45664 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741860_1043] {}] datanode.DataXceiver(331): 127.0.0.1:36049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45664 dst: /127.0.0.1:36049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:26,570 WARN  [Thread-942 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]
2024-11-22T05:35:26,570 WARN  [IPC Server handler 4 on default port 46791 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-22T05:35:26,571 WARN  [IPC Server handler 4 on default port 46791 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-22T05:35:26,571 WARN  [IPC Server handler 4 on default port 46791 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-22T05:35:26,573 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:26,573 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:26,573 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:26,574 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:26,574 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:26,574 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253724529 with entries=23, filesize=23.09 KB; new WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253726553
2024-11-22T05:35:26,575 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33343:33343)]
2024-11-22T05:35:26,575 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253705361 is not closed yet, will try archiving it next time
2024-11-22T05:35:26,575 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253724529 is not closed yet, will try archiving it next time
2024-11-22T05:35:26,575 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253718479 to hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/oldWALs/82891a793a29%2C35099%2C1732253704679.1732253718479
2024-11-22T05:35:26,575 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741846_1029 (size=23654)
2024-11-22T05:35:26,577 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253722492 to hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/oldWALs/82891a793a29%2C35099%2C1732253704679.1732253722492
2024-11-22T05:35:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35099 {}] regionserver.HRegion(8855): Flush requested on 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:26,601 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 767b8467d7ba6cb07f05436770f142ec 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-22T05:35:26,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/749dd5e438064ffd893545982316a430 is 1079, key is tmprow/info:/1732253726600/Put/seqid=0
2024-11-22T05:35:26,609 WARN  [Thread-948 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41959
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:26,609 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45682 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741862_1045] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8]'}, localName='127.0.0.1:36049', datanodeUuid='997782d0-6ecf-48b6-aea7-d2c60565a4a7', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741862_1045 to mirror 127.0.0.1:41959
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:26,609 WARN  [Thread-948 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK], DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]) is bad.
2024-11-22T05:35:26,609 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45682 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741862_1045] {}] datanode.BlockReceiver(316): Block 1073741862 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T05:35:26,609 WARN  [Thread-948 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741862_1045
2024-11-22T05:35:26,609 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45682 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741862_1045] {}] datanode.DataXceiver(331): 127.0.0.1:36049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45682 dst: /127.0.0.1:36049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:26,610 WARN  [Thread-948 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]
2024-11-22T05:35:26,612 WARN  [Thread-948 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:26,612 WARN  [Thread-948 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK], DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]) is bad.
2024-11-22T05:35:26,612 WARN  [Thread-948 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741863_1046
2024-11-22T05:35:26,613 WARN  [Thread-948 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]
2024-11-22T05:35:26,614 WARN  [Thread-948 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:26,615 WARN  [Thread-948 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK], DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]) is bad.
2024-11-22T05:35:26,615 WARN  [Thread-948 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741864_1047
2024-11-22T05:35:26,615 WARN  [Thread-948 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]
2024-11-22T05:35:26,617 WARN  [Thread-948 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:26,617 WARN  [Thread-948 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK], DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]) is bad.
2024-11-22T05:35:26,617 WARN  [Thread-948 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741865_1048
2024-11-22T05:35:26,618 WARN  [Thread-948 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]
2024-11-22T05:35:26,618 WARN  [IPC Server handler 1 on default port 46791 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-22T05:35:26,618 WARN  [IPC Server handler 1 on default port 46791 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-22T05:35:26,618 WARN  [IPC Server handler 1 on default port 46791 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-22T05:35:26,621 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741866_1049 (size=6027)
2024-11-22T05:35:26,829 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:26,976 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253705361 is not closed yet, will try archiving it next time
2024-11-22T05:35:27,022 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/749dd5e438064ffd893545982316a430
2024-11-22T05:35:27,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/749dd5e438064ffd893545982316a430 as hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/749dd5e438064ffd893545982316a430
2024-11-22T05:35:27,042 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/749dd5e438064ffd893545982316a430, entries=1, sequenceid=34, filesize=5.9 K
2024-11-22T05:35:27,043 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 767b8467d7ba6cb07f05436770f142ec in 442ms, sequenceid=34, compaction requested=true
2024-11-22T05:35:27,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 767b8467d7ba6cb07f05436770f142ec:

2024-11-22T05:35:27,043 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-11-22T05:35:27,043 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:35:27,043 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/b6595af9348041e4a438b01eb70d0b43 because midkey is the same as first or last row
2024-11-22T05:35:27,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 767b8467d7ba6cb07f05436770f142ec:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T05:35:27,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:35:27,044 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T05:35:27,045 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T05:35:27,045 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.HStore(1541): 767b8467d7ba6cb07f05436770f142ec/info is initiating minor compaction (all files)
2024-11-22T05:35:27,045 INFO  [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 767b8467d7ba6cb07f05436770f142ec/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.
2024-11-22T05:35:27,046 INFO  [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/d704b6acfbc94fa599eefbdb4da3281d, hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/b6595af9348041e4a438b01eb70d0b43, hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/749dd5e438064ffd893545982316a430] into tmpdir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp, totalSize=28.2 K
2024-11-22T05:35:27,046 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] compactions.Compactor(225): Compacting d704b6acfbc94fa599eefbdb4da3281d, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732253720503
2024-11-22T05:35:27,046 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6595af9348041e4a438b01eb70d0b43, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1732253724558
2024-11-22T05:35:27,047 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] compactions.Compactor(225): Compacting 749dd5e438064ffd893545982316a430, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732253726600
2024-11-22T05:35:27,061 INFO  [RS:0;82891a793a29:35099-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 767b8467d7ba6cb07f05436770f142ec#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T05:35:27,062 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/741aad78ec2540fe9cdc907d92ec80b1 is 1080, key is row0002/info:/1732253720503/Put/seqid=0
2024-11-22T05:35:27,065 WARN  [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41959
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:27,065 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45718 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741867_1050] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8]'}, localName='127.0.0.1:36049', datanodeUuid='997782d0-6ecf-48b6-aea7-d2c60565a4a7', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741867_1050 to mirror 127.0.0.1:41959
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:27,065 WARN  [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK], DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]) is bad.
2024-11-22T05:35:27,065 WARN  [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741867_1050
2024-11-22T05:35:27,065 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45718 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741867_1050] {}] datanode.BlockReceiver(316): Block 1073741867 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T05:35:27,065 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45718 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741867_1050] {}] datanode.DataXceiver(331): 127.0.0.1:36049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45718 dst: /127.0.0.1:36049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:27,066 WARN  [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]
2024-11-22T05:35:27,067 WARN  [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:27,068 WARN  [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK], DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]) is bad.
2024-11-22T05:35:27,068 WARN  [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741868_1051
2024-11-22T05:35:27,068 WARN  [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]
2024-11-22T05:35:27,070 WARN  [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:27,070 WARN  [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK], DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]) is bad.
2024-11-22T05:35:27,070 WARN  [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741869_1052
2024-11-22T05:35:27,071 WARN  [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]
2024-11-22T05:35:27,073 WARN  [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41019
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:27,073 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45730 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741870_1053] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8]'}, localName='127.0.0.1:36049', datanodeUuid='997782d0-6ecf-48b6-aea7-d2c60565a4a7', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741870_1053 to mirror 127.0.0.1:41019
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:27,073 WARN  [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK], DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]) is bad.
2024-11-22T05:35:27,073 WARN  [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741870_1053
2024-11-22T05:35:27,074 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45730 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741870_1053] {}] datanode.BlockReceiver(316): Block 1073741870 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T05:35:27,074 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45730 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741870_1053] {}] datanode.DataXceiver(331): 127.0.0.1:36049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45730 dst: /127.0.0.1:36049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:27,074 WARN  [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]
2024-11-22T05:35:27,075 WARN  [IPC Server handler 0 on default port 46791 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-22T05:35:27,075 WARN  [IPC Server handler 0 on default port 46791 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-22T05:35:27,075 WARN  [IPC Server handler 0 on default port 46791 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-22T05:35:27,078 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741871_1054 (size=17994)
2024-11-22T05:35:27,283 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@b3c3ffa[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36049, datanodeUuid=997782d0-6ecf-48b6-aea7-d2c60565a4a7, infoPort=33343, infoSecurePort=0, ipcPort=45673, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022):Failed to transfer BP-860069939-172.17.0.2-1732253702022:blk_1073741851_1034 to 127.0.0.1:44489 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:27,283 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5fbd135[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36049, datanodeUuid=997782d0-6ecf-48b6-aea7-d2c60565a4a7, infoPort=33343, infoSecurePort=0, ipcPort=45673, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022):Failed to transfer BP-860069939-172.17.0.2-1732253702022:blk_1073741841_1024 to 127.0.0.1:44489 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:27,487 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/741aad78ec2540fe9cdc907d92ec80b1 as hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/741aad78ec2540fe9cdc907d92ec80b1
2024-11-22T05:35:27,496 INFO  [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 767b8467d7ba6cb07f05436770f142ec/info of 767b8467d7ba6cb07f05436770f142ec into 741aad78ec2540fe9cdc907d92ec80b1(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T05:35:27,497 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 767b8467d7ba6cb07f05436770f142ec:

2024-11-22T05:35:27,497 INFO  [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec., storeName=767b8467d7ba6cb07f05436770f142ec/info, priority=13, startTime=1732253727043; duration=0sec
2024-11-22T05:35:27,497 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-22T05:35:27,497 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:35:27,497 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/741aad78ec2540fe9cdc907d92ec80b1 because midkey is the same as first or last row
2024-11-22T05:35:27,497 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-22T05:35:27,497 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:35:27,497 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/741aad78ec2540fe9cdc907d92ec80b1 because midkey is the same as first or last row
2024-11-22T05:35:27,498 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-22T05:35:27,498 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:35:27,498 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/741aad78ec2540fe9cdc907d92ec80b1 because midkey is the same as first or last row
2024-11-22T05:35:27,498 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:35:27,498 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 767b8467d7ba6cb07f05436770f142ec:info
2024-11-22T05:35:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35099 {}] regionserver.HRegion(8855): Flush requested on 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:28,025 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 767b8467d7ba6cb07f05436770f142ec 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-22T05:35:28,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/087bcae27fea41aab895dff4af459e2b is 1079, key is tmprow/info:/1732253728023/Put/seqid=0
2024-11-22T05:35:28,035 WARN  [Thread-965 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:28,035 WARN  [Thread-965 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK], DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]) is bad.
2024-11-22T05:35:28,035 WARN  [Thread-965 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741872_1055
2024-11-22T05:35:28,036 WARN  [Thread-965 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41959,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK]
2024-11-22T05:35:28,038 WARN  [Thread-965 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:28,038 WARN  [Thread-965 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK], DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]) is bad.
2024-11-22T05:35:28,038 WARN  [Thread-965 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741873_1056
2024-11-22T05:35:28,039 WARN  [Thread-965 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]
2024-11-22T05:35:28,044 WARN  [Thread-965 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44683
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:28,044 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45746 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741874_1057] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8]'}, localName='127.0.0.1:36049', datanodeUuid='997782d0-6ecf-48b6-aea7-d2c60565a4a7', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741874_1057 to mirror 127.0.0.1:44683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:28,044 WARN  [Thread-965 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK], DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]) is bad.
2024-11-22T05:35:28,044 WARN  [Thread-965 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741874_1057
2024-11-22T05:35:28,044 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45746 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741874_1057] {}] datanode.BlockReceiver(316): Block 1073741874 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T05:35:28,044 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45746 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741874_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45746 dst: /127.0.0.1:36049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:28,045 WARN  [Thread-965 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]
2024-11-22T05:35:28,047 WARN  [Thread-965 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44489
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:28,047 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45750 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741875_1058] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8]'}, localName='127.0.0.1:36049', datanodeUuid='997782d0-6ecf-48b6-aea7-d2c60565a4a7', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741875_1058 to mirror 127.0.0.1:44489
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:28,048 WARN  [Thread-965 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK], DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]) is bad.
2024-11-22T05:35:28,048 WARN  [Thread-965 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741875_1058
2024-11-22T05:35:28,048 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45750 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741875_1058] {}] datanode.BlockReceiver(316): Block 1073741875 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T05:35:28,048 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:45750 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741875_1058] {}] datanode.DataXceiver(331): 127.0.0.1:36049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45750 dst: /127.0.0.1:36049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:28,048 WARN  [Thread-965 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44489,DS-181c5212-0c5a-4745-b234-bc5e6713eebb,DISK]
2024-11-22T05:35:28,049 WARN  [IPC Server handler 2 on default port 46791 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-22T05:35:28,049 WARN  [IPC Server handler 2 on default port 46791 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-22T05:35:28,049 WARN  [IPC Server handler 2 on default port 46791 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-22T05:35:28,053 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741876_1059 (size=6027)
2024-11-22T05:35:28,177 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:28,283 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5fbd135[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36049, datanodeUuid=997782d0-6ecf-48b6-aea7-d2c60565a4a7, infoPort=33343, infoSecurePort=0, ipcPort=45673, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022):Failed to transfer BP-860069939-172.17.0.2-1732253702022:blk_1073741856_1039 to 127.0.0.1:41019 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:28,283 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@b3c3ffa[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36049, datanodeUuid=997782d0-6ecf-48b6-aea7-d2c60565a4a7, infoPort=33343, infoSecurePort=0, ipcPort=45673, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022):Failed to transfer BP-860069939-172.17.0.2-1732253702022:blk_1073741846_1029 to 127.0.0.1:44683 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:28,454 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/087bcae27fea41aab895dff4af459e2b
2024-11-22T05:35:28,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/087bcae27fea41aab895dff4af459e2b as hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/087bcae27fea41aab895dff4af459e2b
2024-11-22T05:35:28,471 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/087bcae27fea41aab895dff4af459e2b, entries=1, sequenceid=45, filesize=5.9 K
2024-11-22T05:35:28,472 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 767b8467d7ba6cb07f05436770f142ec in 447ms, sequenceid=45, compaction requested=false
2024-11-22T05:35:28,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 767b8467d7ba6cb07f05436770f142ec:

2024-11-22T05:35:28,472 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-11-22T05:35:28,472 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:35:28,472 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/741aad78ec2540fe9cdc907d92ec80b1 because midkey is the same as first or last row
2024-11-22T05:35:28,575 WARN  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-11-22T05:35:28,575 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:28,644 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:35:28,649 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:35:28,649 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:35:28,650 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:35:28,650 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:35:28,650 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ecaf507{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:35:28,651 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@40e683d1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:35:28,757 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@16711f95{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/java.io.tmpdir/jetty-localhost-35267-hadoop-hdfs-3_4_1-tests_jar-_-any-17787242041193624518/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:35:28,758 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4595b5c3{HTTP/1.1, (http/1.1)}{localhost:35267}
2024-11-22T05:35:28,758 INFO  [Time-limited test {}] server.Server(415): Started @136015ms
2024-11-22T05:35:28,759 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:35:28,829 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:29,217 WARN  [Thread-985 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:35:29,225 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x342869674651facf with lease ID 0x2f19ab655b3c7ec1: from storage DS-af01850a-5d72-4c8b-a217-d2df8ee2427d node DatanodeRegistration(127.0.0.1:42715, datanodeUuid=7022ae67-a4d8-42d1-b7a4-82fbd184c97e, infoPort=43729, infoSecurePort=0, ipcPort=36417, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:35:29,225 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x342869674651facf with lease ID 0x2f19ab655b3c7ec1: from storage DS-4f178493-56f5-4235-9403-d0648c301e42 node DatanodeRegistration(127.0.0.1:42715, datanodeUuid=7022ae67-a4d8-42d1-b7a4-82fbd184c97e, infoPort=43729, infoSecurePort=0, ipcPort=36417, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-22T05:35:30,177 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:30,284 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@b3c3ffa[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36049, datanodeUuid=997782d0-6ecf-48b6-aea7-d2c60565a4a7, infoPort=33343, infoSecurePort=0, ipcPort=45673, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022):Failed to transfer BP-860069939-172.17.0.2-1732253702022:blk_1073741866_1049 to 127.0.0.1:44683 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:30,284 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5fbd135[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36049, datanodeUuid=997782d0-6ecf-48b6-aea7-d2c60565a4a7, infoPort=33343, infoSecurePort=0, ipcPort=45673, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022):Failed to transfer BP-860069939-172.17.0.2-1732253702022:blk_1073741871_1054 to 127.0.0.1:44489 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:30,576 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:30,829 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:31,284 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5fbd135[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36049, datanodeUuid=997782d0-6ecf-48b6-aea7-d2c60565a4a7, infoPort=33343, infoSecurePort=0, ipcPort=45673, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022):Failed to transfer BP-860069939-172.17.0.2-1732253702022:blk_1073741876_1059 to 127.0.0.1:44489 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:32,178 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:32,576 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:32,830 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:34,178 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:34,462 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-22T05:35:34,576 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:34,830 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:35,132 ERROR [FSHLog-0-hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData-prefix:82891a793a29,36973,1732253704491 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:35,133 WARN  [FSHLog-0-hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData-prefix:82891a793a29,36973,1732253704491 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:35,133 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 82891a793a29%2C36973%2C1732253704491:(num 1732253704853) roll requested
2024-11-22T05:35:35,134 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C36973%2C1732253704491.1732253735133
2024-11-22T05:35:35,137 WARN  [Thread-1005 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44683
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:35,137 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-358693681_22 at /127.0.0.1:60174 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741877_1060] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data4]'}, localName='127.0.0.1:42715', datanodeUuid='7022ae67-a4d8-42d1-b7a4-82fbd184c97e', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741877_1060 to mirror 127.0.0.1:44683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:35,138 WARN  [Thread-1005 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42715,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK], DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]) is bad.
2024-11-22T05:35:35,138 WARN  [Thread-1005 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741877_1060
2024-11-22T05:35:35,138 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-358693681_22 at /127.0.0.1:60174 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741877_1060] {}] datanode.BlockReceiver(316): Block 1073741877 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-22T05:35:35,138 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-358693681_22 at /127.0.0.1:60174 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741877_1060] {}] datanode.DataXceiver(331): 127.0.0.1:42715:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60174 dst: /127.0.0.1:42715
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:35,138 WARN  [Thread-1005 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]
2024-11-22T05:35:35,145 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:35,145 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:35,145 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:35,145 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:35,145 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:35,145 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/WALs/82891a793a29,36973,1732253704491/82891a793a29%2C36973%2C1732253704491.1732253704853 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/WALs/82891a793a29,36973,1732253704491/82891a793a29%2C36973%2C1732253704491.1732253735133
2024-11-22T05:35:35,146 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:35,146 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:35,146 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/WALs/82891a793a29,36973,1732253704491/82891a793a29%2C36973%2C1732253704491.1732253704853
2024-11-22T05:35:35,146 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43729:43729),(127.0.0.1/127.0.0.1:33343:33343)]
2024-11-22T05:35:35,146 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/WALs/82891a793a29,36973,1732253704491/82891a793a29%2C36973%2C1732253704491.1732253704853 is not closed yet, will try archiving it next time
2024-11-22T05:35:35,146 WARN  [IPC Server handler 4 on default port 46791 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/WALs/82891a793a29,36973,1732253704491/82891a793a29%2C36973%2C1732253704491.1732253704853 has not been closed. Lease recovery is in progress. RecoveryId = 1062 for block blk_1073741830_1006
2024-11-22T05:35:35,147 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/WALs/82891a793a29,36973,1732253704491/82891a793a29%2C36973%2C1732253704491.1732253704853 after 1ms
2024-11-22T05:35:36,179 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:36,577 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:38,179 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:38,577 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:39,227 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/WALs/82891a793a29,36973,1732253704491/82891a793a29%2C36973%2C1732253704491.1732253704853 after 4081ms
2024-11-22T05:35:39,240 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@72421f62 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-860069939-172.17.0.2-1732253702022:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:41019,null,null])
java.net.ConnectException: Call From 82891a793a29/172.17.0.2 to localhost:34853 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-22T05:35:39,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741833_1019 (size=455)
2024-11-22T05:35:39,515 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253705361 to hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/oldWALs/82891a793a29%2C35099%2C1732253704679.1732253705361
2024-11-22T05:35:39,517 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253724529 to hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/oldWALs/82891a793a29%2C35099%2C1732253704679.1732253724529
2024-11-22T05:35:40,180 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:40,578 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:42,180 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:42,222 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741833_1019 (size=455)
2024-11-22T05:35:42,578 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:43,913 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C35099%2C1732253704679.1732253743913
2024-11-22T05:35:43,918 WARN  [Thread-1018 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1063
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44683
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:43,918 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-358693681_22 at /127.0.0.1:42458 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741879_1063] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data4]'}, localName='127.0.0.1:42715', datanodeUuid='7022ae67-a4d8-42d1-b7a4-82fbd184c97e', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741879_1063 to mirror 127.0.0.1:44683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:43,919 WARN  [Thread-1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741879_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42715,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK], DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]) is bad.
2024-11-22T05:35:43,919 WARN  [Thread-1018 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741879_1063
2024-11-22T05:35:43,919 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-358693681_22 at /127.0.0.1:42458 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741879_1063] {}] datanode.BlockReceiver(316): Block 1073741879 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-22T05:35:43,919 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-358693681_22 at /127.0.0.1:42458 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741879_1063] {}] datanode.DataXceiver(331): 127.0.0.1:42715:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42458 dst: /127.0.0.1:42715
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:43,920 WARN  [Thread-1018 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]
2024-11-22T05:35:43,925 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:43,926 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:43,926 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:43,926 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:43,926 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:43,926 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253726553 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253743913
2024-11-22T05:35:43,927 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43729:43729),(127.0.0.1/127.0.0.1:33343:33343)]
2024-11-22T05:35:43,927 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253726553 is not closed yet, will try archiving it next time
2024-11-22T05:35:43,928 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741861_1044 (size=13591)
2024-11-22T05:35:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35099 {}] regionserver.HRegion(8855): Flush requested on 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:43,940 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 767b8467d7ba6cb07f05436770f142ec 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-22T05:35:43,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/55766cd800264572a46bc82d8b96e622 is 1080, key is row0013/info:/1732253743929/Put/seqid=0
2024-11-22T05:35:43,949 WARN  [Thread-1025 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1065
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44683
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:43,948 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:53284 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741881_1065] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8]'}, localName='127.0.0.1:36049', datanodeUuid='997782d0-6ecf-48b6-aea7-d2c60565a4a7', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741881_1065 to mirror 127.0.0.1:44683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:43,949 WARN  [Thread-1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741881_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK], DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]) is bad.
2024-11-22T05:35:43,949 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:53284 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741881_1065] {}] datanode.BlockReceiver(316): Block 1073741881 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T05:35:43,949 WARN  [Thread-1025 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741881_1065
2024-11-22T05:35:43,949 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:53284 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741881_1065] {}] datanode.DataXceiver(331): 127.0.0.1:36049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53284 dst: /127.0.0.1:36049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:43,950 WARN  [Thread-1025 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]
2024-11-22T05:35:43,956 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741882_1066 (size=11421)
2024-11-22T05:35:43,956 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741882_1066 (size=11421)
2024-11-22T05:35:43,957 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/55766cd800264572a46bc82d8b96e622
2024-11-22T05:35:43,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/55766cd800264572a46bc82d8b96e622 as hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/55766cd800264572a46bc82d8b96e622
2024-11-22T05:35:43,970 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/55766cd800264572a46bc82d8b96e622, entries=6, sequenceid=55, filesize=11.2 K
2024-11-22T05:35:43,972 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7530, heapSize ~8.11 KB/8304, currentSize=6.30 KB/6455 for 767b8467d7ba6cb07f05436770f142ec in 32ms, sequenceid=55, compaction requested=true
2024-11-22T05:35:43,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 767b8467d7ba6cb07f05436770f142ec:

2024-11-22T05:35:43,972 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=34.6 K, sizeToCheck=16.0 K
2024-11-22T05:35:43,972 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:35:43,972 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/741aad78ec2540fe9cdc907d92ec80b1 because midkey is the same as first or last row
2024-11-22T05:35:43,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 767b8467d7ba6cb07f05436770f142ec:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T05:35:43,972 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T05:35:43,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:35:43,974 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 35442 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T05:35:43,974 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.HStore(1541): 767b8467d7ba6cb07f05436770f142ec/info is initiating minor compaction (all files)
2024-11-22T05:35:43,974 INFO  [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 767b8467d7ba6cb07f05436770f142ec/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.
2024-11-22T05:35:43,974 INFO  [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/741aad78ec2540fe9cdc907d92ec80b1, hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/087bcae27fea41aab895dff4af459e2b, hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/55766cd800264572a46bc82d8b96e622] into tmpdir=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp, totalSize=34.6 K
2024-11-22T05:35:43,975 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] compactions.Compactor(225): Compacting 741aad78ec2540fe9cdc907d92ec80b1, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732253720503
2024-11-22T05:35:43,975 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] compactions.Compactor(225): Compacting 087bcae27fea41aab895dff4af459e2b, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732253728023
2024-11-22T05:35:43,975 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55766cd800264572a46bc82d8b96e622, keycount=6, bloomtype=ROW, size=11.2 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732253728432
2024-11-22T05:35:43,994 INFO  [RS:0;82891a793a29:35099-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 767b8467d7ba6cb07f05436770f142ec#info#compaction#24 average throughput is 5.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T05:35:43,994 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/d930a5c6e0c24c329b216f5b68549b04 is 1080, key is row0002/info:/1732253720503/Put/seqid=0
2024-11-22T05:35:43,997 WARN  [Thread-1036 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1067
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44683
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:43,997 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:53320 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741883_1067] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8]'}, localName='127.0.0.1:36049', datanodeUuid='997782d0-6ecf-48b6-aea7-d2c60565a4a7', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741883_1067 to mirror 127.0.0.1:44683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:43,997 WARN  [Thread-1036 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741883_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK], DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]) is bad.
2024-11-22T05:35:43,998 WARN  [Thread-1036 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741883_1067
2024-11-22T05:35:43,998 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:53320 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741883_1067] {}] datanode.BlockReceiver(316): Block 1073741883 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T05:35:43,998 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:53320 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741883_1067] {}] datanode.DataXceiver(331): 127.0.0.1:36049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53320 dst: /127.0.0.1:36049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:43,998 WARN  [Thread-1036 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]
2024-11-22T05:35:44,004 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741884_1068 (size=23502)
2024-11-22T05:35:44,004 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741884_1068 (size=23502)
2024-11-22T05:35:44,011 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/d930a5c6e0c24c329b216f5b68549b04 as hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/d930a5c6e0c24c329b216f5b68549b04
2024-11-22T05:35:44,019 INFO  [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 767b8467d7ba6cb07f05436770f142ec/info of 767b8467d7ba6cb07f05436770f142ec into d930a5c6e0c24c329b216f5b68549b04(size=23.0 K), total size for store is 23.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T05:35:44,019 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 767b8467d7ba6cb07f05436770f142ec:

2024-11-22T05:35:44,019 INFO  [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec., storeName=767b8467d7ba6cb07f05436770f142ec/info, priority=13, startTime=1732253743972; duration=0sec
2024-11-22T05:35:44,019 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.0 K, sizeToCheck=16.0 K
2024-11-22T05:35:44,019 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:35:44,019 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/d930a5c6e0c24c329b216f5b68549b04 because midkey is the same as first or last row
2024-11-22T05:35:44,019 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.0 K, sizeToCheck=16.0 K
2024-11-22T05:35:44,019 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:35:44,019 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/d930a5c6e0c24c329b216f5b68549b04 because midkey is the same as first or last row
2024-11-22T05:35:44,019 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.0 K, sizeToCheck=16.0 K
2024-11-22T05:35:44,019 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:35:44,019 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/d930a5c6e0c24c329b216f5b68549b04 because midkey is the same as first or last row
2024-11-22T05:35:44,019 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:35:44,020 DEBUG [RS:0;82891a793a29:35099-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 767b8467d7ba6cb07f05436770f142ec:info
2024-11-22T05:35:44,180 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35099 {}] regionserver.HRegion(8855): Flush requested on 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:44,185 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 767b8467d7ba6cb07f05436770f142ec 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-22T05:35:44,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/3e3305573472487385f4f2a45eea6b27 is 1080, key is row0018/info:/1732253743942/Put/seqid=0
2024-11-22T05:35:44,202 WARN  [Thread-1043 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1069
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44683
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:44,202 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:42500 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741885_1069] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data4]'}, localName='127.0.0.1:42715', datanodeUuid='7022ae67-a4d8-42d1-b7a4-82fbd184c97e', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741885_1069 to mirror 127.0.0.1:44683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:44,202 WARN  [Thread-1043 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741885_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42715,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK], DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]) is bad.
2024-11-22T05:35:44,202 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:42500 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741885_1069] {}] datanode.BlockReceiver(316): Block 1073741885 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T05:35:44,202 WARN  [Thread-1043 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741885_1069
2024-11-22T05:35:44,203 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:42500 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741885_1069] {}] datanode.DataXceiver(331): 127.0.0.1:42715:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42500 dst: /127.0.0.1:42715
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:44,203 WARN  [Thread-1043 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]
2024-11-22T05:35:44,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741886_1070 (size=11421)
2024-11-22T05:35:44,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741886_1070 (size=11421)
2024-11-22T05:35:44,209 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/3e3305573472487385f4f2a45eea6b27
2024-11-22T05:35:44,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/3e3305573472487385f4f2a45eea6b27 as hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/3e3305573472487385f4f2a45eea6b27
2024-11-22T05:35:44,222 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/3e3305573472487385f4f2a45eea6b27, entries=6, sequenceid=66, filesize=11.2 K
2024-11-22T05:35:44,223 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7530, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 767b8467d7ba6cb07f05436770f142ec in 38ms, sequenceid=66, compaction requested=false
2024-11-22T05:35:44,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 767b8467d7ba6cb07f05436770f142ec:

2024-11-22T05:35:44,224 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=34.1 K, sizeToCheck=16.0 K
2024-11-22T05:35:44,224 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:35:44,224 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/d930a5c6e0c24c329b216f5b68549b04 because midkey is the same as first or last row
2024-11-22T05:35:44,329 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.1732253726553 to hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/oldWALs/82891a793a29%2C35099%2C1732253704679.1732253726553
2024-11-22T05:35:44,579 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-11-22T05:35:44,579 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:44,589 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-22T05:35:44,589 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T05:35:44,589 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:35:44,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:35:44,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:35:44,589 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-22T05:35:44,589 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-22T05:35:44,590 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=821688174, stopped=false
2024-11-22T05:35:44,590 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=82891a793a29,36973,1732253704491
2024-11-22T05:35:44,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T05:35:44,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T05:35:44,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33487-0x1016109dcbc0002, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T05:35:44,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33487-0x1016109dcbc0002, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:44,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:44,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:44,635 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T05:35:44,635 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T05:35:44,635 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:35:44,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:35:44,636 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '82891a793a29,35099,1732253704679' *****
2024-11-22T05:35:44,636 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-22T05:35:44,636 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '82891a793a29,33487,1732253706069' *****
2024-11-22T05:35:44,636 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-22T05:35:44,636 INFO  [RS:0;82891a793a29:35099 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-22T05:35:44,636 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:35:44,636 INFO  [RS:1;82891a793a29:33487 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-22T05:35:44,636 INFO  [RS:0;82891a793a29:35099 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-22T05:35:44,636 INFO  [RS:0;82891a793a29:35099 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-22T05:35:44,636 INFO  [RS:1;82891a793a29:33487 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-22T05:35:44,636 INFO  [RS:1;82891a793a29:33487 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-22T05:35:44,636 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:35:44,636 INFO  [RS:1;82891a793a29:33487 {}] regionserver.HRegionServer(959): stopping server 82891a793a29,33487,1732253706069
2024-11-22T05:35:44,636 INFO  [RS:1;82891a793a29:33487 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T05:35:44,636 INFO  [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(3091): Received CLOSE for 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:44,637 INFO  [RS:1;82891a793a29:33487 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;82891a793a29:33487.
2024-11-22T05:35:44,637 DEBUG [RS:1;82891a793a29:33487 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:35:44,637 DEBUG [RS:1;82891a793a29:33487 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:35:44,637 INFO  [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(959): stopping server 82891a793a29,35099,1732253704679
2024-11-22T05:35:44,637 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33487-0x1016109dcbc0002, quorum=127.0.0.1:59485, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:35:44,637 INFO  [RS:0;82891a793a29:35099 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T05:35:44,637 INFO  [RS:1;82891a793a29:33487 {}] regionserver.HRegionServer(976): stopping server 82891a793a29,33487,1732253706069; all regions closed.
2024-11-22T05:35:44,637 INFO  [RS:0;82891a793a29:35099 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;82891a793a29:35099.
2024-11-22T05:35:44,637 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 767b8467d7ba6cb07f05436770f142ec, disabling compactions & flushes
2024-11-22T05:35:44,637 DEBUG [RS:0;82891a793a29:35099 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:35:44,637 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.
2024-11-22T05:35:44,637 DEBUG [RS:0;82891a793a29:35099 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:35:44,637 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-22T05:35:44,637 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.
2024-11-22T05:35:44,637 INFO  [RS:0;82891a793a29:35099 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-22T05:35:44,637 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec. after waiting 0 ms
2024-11-22T05:35:44,637 INFO  [RS:0;82891a793a29:35099 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-22T05:35:44,637 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.
2024-11-22T05:35:44,637 INFO  [RS:0;82891a793a29:35099 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-22T05:35:44,637 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-22T05:35:44,637 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:44,637 INFO  [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-22T05:35:44,638 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 767b8467d7ba6cb07f05436770f142ec 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-22T05:35:44,638 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:44,638 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:44,638 INFO  [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-22T05:35:44,638 DEBUG [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 767b8467d7ba6cb07f05436770f142ec=TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.}
2024-11-22T05:35:44,638 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:44,638 DEBUG [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:44,638 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T05:35:44,638 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:44,638 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T05:35:44,638 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T05:35:44,638 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T05:35:44,638 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T05:35:44,638 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-11-22T05:35:44,639 ERROR [FSHLog-0-hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253-prefix:82891a793a29,35099,1732253704679.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:44,639 WARN  [FSHLog-0-hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253-prefix:82891a793a29,35099,1732253704679.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:44,639 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 82891a793a29%2C35099%2C1732253704679.meta:.meta(num 1732253705835) roll requested
2024-11-22T05:35:44,639 INFO  [regionserver/82891a793a29:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C35099%2C1732253704679.meta.1732253744639.meta
2024-11-22T05:35:44,644 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:44,644 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:44,644 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
2024-11-22T05:35:44,645 WARN  [IPC Server handler 4 on default port 46791 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319 has not been closed. Lease recovery is in progress. RecoveryId = 1071 for block blk_1073741837_1013
2024-11-22T05:35:44,645 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319 after 1ms
2024-11-22T05:35:44,647 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/cbae7baa6f6f4038b1ba6d524f2bc65c is 1079, key is tmprow/info:/1732253744387/Put/seqid=0
2024-11-22T05:35:44,648 WARN  [Thread-1050 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1072
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44683
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:44,648 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:53346 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741887_1072] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8]'}, localName='127.0.0.1:36049', datanodeUuid='997782d0-6ecf-48b6-aea7-d2c60565a4a7', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741887_1072 to mirror 127.0.0.1:44683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:44,648 WARN  [Thread-1050 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741887_1072 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK], DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]) is bad.
2024-11-22T05:35:44,648 WARN  [Thread-1050 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741887_1072
2024-11-22T05:35:44,648 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:53346 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741887_1072] {}] datanode.BlockReceiver(316): Block 1073741887 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-22T05:35:44,648 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:53346 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741887_1072] {}] datanode.DataXceiver(331): 127.0.0.1:36049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53346 dst: /127.0.0.1:36049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:44,648 WARN  [Thread-1050 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]
2024-11-22T05:35:44,648 WARN  [Thread-1051 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1073
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:44,649 WARN  [Thread-1051 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741888_1073 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK], DatanodeInfoWithStorage[127.0.0.1:36049,DS-3e0c0083-25dc-4297-bc52-9a1ba10cad01,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]) is bad.
2024-11-22T05:35:44,649 WARN  [Thread-1051 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741888_1073
2024-11-22T05:35:44,649 WARN  [Thread-1051 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]
2024-11-22T05:35:44,653 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:44,654 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:44,654 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:44,654 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:44,654 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:44,654 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253744639.meta
2024-11-22T05:35:44,655 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:44,655 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41019,DS-6fc5753a-62f0-48d5-88c3-196b62da3e5e,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:44,655 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
2024-11-22T05:35:44,655 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43729:43729),(127.0.0.1/127.0.0.1:33343:33343)]
2024-11-22T05:35:44,655 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta is not closed yet, will try archiving it next time
2024-11-22T05:35:44,655 WARN  [IPC Server handler 4 on default port 46791 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta has not been closed. Lease recovery is in progress. RecoveryId = 1076 for block blk_1073741834_1010
2024-11-22T05:35:44,656 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta after 1ms
2024-11-22T05:35:44,656 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741890_1075 (size=6027)
2024-11-22T05:35:44,656 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741890_1075 (size=6027)
2024-11-22T05:35:44,671 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/.tmp/info/4069d7a11fcf421ea2eb0bd48a015fc0 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec./info:regioninfo/1732253706602/Put/seqid=0
2024-11-22T05:35:44,676 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741891_1077 (size=7089)
2024-11-22T05:35:44,676 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741891_1077 (size=7089)
2024-11-22T05:35:44,677 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/.tmp/info/4069d7a11fcf421ea2eb0bd48a015fc0
2024-11-22T05:35:44,698 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/.tmp/ns/1780efc94493419a88cdb16057f9c0fa is 43, key is default/ns:d/1732253705922/Put/seqid=0
2024-11-22T05:35:44,701 WARN  [Thread-1069 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741892_1078
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44683
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:35:44,701 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:42544 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741892_1078] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data4]'}, localName='127.0.0.1:42715', datanodeUuid='7022ae67-a4d8-42d1-b7a4-82fbd184c97e', xmitsInProgress=0}:Exception transferring block BP-860069939-172.17.0.2-1732253702022:blk_1073741892_1078 to mirror 127.0.0.1:44683
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:44,702 WARN  [Thread-1069 {}] hdfs.DataStreamer(1731): Error Recovery for BP-860069939-172.17.0.2-1732253702022:blk_1073741892_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42715,DS-af01850a-5d72-4c8b-a217-d2df8ee2427d,DISK], DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]) is bad.
2024-11-22T05:35:44,702 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:42544 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741892_1078] {}] datanode.BlockReceiver(316): Block 1073741892 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T05:35:44,702 WARN  [Thread-1069 {}] hdfs.DataStreamer(1850): Abandoning BP-860069939-172.17.0.2-1732253702022:blk_1073741892_1078
2024-11-22T05:35:44,702 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-112347847_22 at /127.0.0.1:42544 [Receiving block BP-860069939-172.17.0.2-1732253702022:blk_1073741892_1078] {}] datanode.DataXceiver(331): 127.0.0.1:42715:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42544 dst: /127.0.0.1:42715
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:44,702 WARN  [Thread-1069 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44683,DS-b3001d0f-cf6c-4fdf-af56-cee7a5ca348b,DISK]
2024-11-22T05:35:44,708 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741893_1079 (size=5153)
2024-11-22T05:35:44,708 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741893_1079 (size=5153)
2024-11-22T05:35:44,838 DEBUG [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:45,039 DEBUG [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 767b8467d7ba6cb07f05436770f142ec
2024-11-22T05:35:45,055 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/cbae7baa6f6f4038b1ba6d524f2bc65c
2024-11-22T05:35:45,062 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/.tmp/info/cbae7baa6f6f4038b1ba6d524f2bc65c as hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/cbae7baa6f6f4038b1ba6d524f2bc65c
2024-11-22T05:35:45,068 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/cbae7baa6f6f4038b1ba6d524f2bc65c, entries=1, sequenceid=70, filesize=5.9 K
2024-11-22T05:35:45,069 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1075, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 767b8467d7ba6cb07f05436770f142ec in 432ms, sequenceid=70, compaction requested=true
2024-11-22T05:35:45,070 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/d704b6acfbc94fa599eefbdb4da3281d, hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/b6595af9348041e4a438b01eb70d0b43, hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/741aad78ec2540fe9cdc907d92ec80b1, hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/749dd5e438064ffd893545982316a430, hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/087bcae27fea41aab895dff4af459e2b, hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/55766cd800264572a46bc82d8b96e622] to archive
2024-11-22T05:35:45,071 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-22T05:35:45,073 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/d704b6acfbc94fa599eefbdb4da3281d to hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/d704b6acfbc94fa599eefbdb4da3281d
2024-11-22T05:35:45,074 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/b6595af9348041e4a438b01eb70d0b43 to hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/b6595af9348041e4a438b01eb70d0b43
2024-11-22T05:35:45,076 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/741aad78ec2540fe9cdc907d92ec80b1 to hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/741aad78ec2540fe9cdc907d92ec80b1
2024-11-22T05:35:45,077 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/749dd5e438064ffd893545982316a430 to hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/749dd5e438064ffd893545982316a430
2024-11-22T05:35:45,078 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/087bcae27fea41aab895dff4af459e2b to hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/087bcae27fea41aab895dff4af459e2b
2024-11-22T05:35:45,080 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/55766cd800264572a46bc82d8b96e622 to hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/info/55766cd800264572a46bc82d8b96e622
2024-11-22T05:35:45,080 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=82891a793a29:36973 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-22T05:35:45,081 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [d704b6acfbc94fa599eefbdb4da3281d=10347, b6595af9348041e4a438b01eb70d0b43=12506, 741aad78ec2540fe9cdc907d92ec80b1=17994, 749dd5e438064ffd893545982316a430=6027, 087bcae27fea41aab895dff4af459e2b=6027, 55766cd800264572a46bc82d8b96e622=11421]
2024-11-22T05:35:45,085 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/default/TestLogRolling-testLogRollOnDatanodeDeath/767b8467d7ba6cb07f05436770f142ec/recovered.edits/73.seqid, newMaxSeqId=73, maxSeqId=1
2024-11-22T05:35:45,086 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.
2024-11-22T05:35:45,086 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 767b8467d7ba6cb07f05436770f142ec:
Waiting for close lock at 1732253744637Running coprocessor pre-close hooks at 1732253744637Disabling compacts and flushes for region at 1732253744637Disabling writes for close at 1732253744637Obtaining lock to block concurrent updates at 1732253744638 (+1 ms)Preparing flush snapshotting stores in 767b8467d7ba6cb07f05436770f142ec at 1732253744638Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec., syncing WAL and waiting on mvcc, flushsize=dataSize=1075, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1732253744638Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec. at 1732253744639 (+1 ms)Flushing 767b8467d7ba6cb07f05436770f142ec/info: creating writer at 1732253744639Flushing 767b8467d7ba6cb07f05436770f142ec/info: appending metadata at 1732253744646 (+7 ms)Flushing 767b8467d7ba6cb07f05436770f142ec/info: closing flushed file at 1732253744646Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5598ebd: reopening flushed file at 1732253745061 (+415 ms)Finished flush of dataSize ~1.05 KB/1075, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 767b8467d7ba6cb07f05436770f142ec in 432ms, sequenceid=70, compaction requested=true at 1732253745069 (+8 ms)Writing region close event to WAL at 1732253745081 (+12 ms)Running coprocessor post-close hooks at 1732253745086 (+5 ms)Closed at 1732253745086
2024-11-22T05:35:45,086 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732253706219.767b8467d7ba6cb07f05436770f142ec.
2024-11-22T05:35:45,109 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/.tmp/ns/1780efc94493419a88cdb16057f9c0fa
2024-11-22T05:35:45,131 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/.tmp/table/c11f50f4f8084e2ea179473ca1bbfa6f is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1732253706616/Put/seqid=0
2024-11-22T05:35:45,136 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741894_1080 (size=5424)
2024-11-22T05:35:45,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741894_1080 (size=5424)
2024-11-22T05:35:45,140 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/.tmp/table/c11f50f4f8084e2ea179473ca1bbfa6f
2024-11-22T05:35:45,147 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/.tmp/info/4069d7a11fcf421ea2eb0bd48a015fc0 as hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/info/4069d7a11fcf421ea2eb0bd48a015fc0
2024-11-22T05:35:45,157 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/info/4069d7a11fcf421ea2eb0bd48a015fc0, entries=10, sequenceid=11, filesize=6.9 K
2024-11-22T05:35:45,158 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/.tmp/ns/1780efc94493419a88cdb16057f9c0fa as hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/ns/1780efc94493419a88cdb16057f9c0fa
2024-11-22T05:35:45,166 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/ns/1780efc94493419a88cdb16057f9c0fa, entries=2, sequenceid=11, filesize=5.0 K
2024-11-22T05:35:45,168 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/.tmp/table/c11f50f4f8084e2ea179473ca1bbfa6f as hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/table/c11f50f4f8084e2ea179473ca1bbfa6f
2024-11-22T05:35:45,175 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/table/c11f50f4f8084e2ea179473ca1bbfa6f, entries=2, sequenceid=11, filesize=5.3 K
2024-11-22T05:35:45,177 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 539ms, sequenceid=11, compaction requested=false
2024-11-22T05:35:45,185 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-22T05:35:45,185 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T05:35:45,185 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T05:35:45,186 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732253744638Running coprocessor pre-close hooks at 1732253744638Disabling compacts and flushes for region at 1732253744638Disabling writes for close at 1732253744638Obtaining lock to block concurrent updates at 1732253744638Preparing flush snapshotting stores in 1588230740 at 1732253744638Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1732253744639 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732253744656 (+17 ms)Flushing 1588230740/info: creating writer at 1732253744656Flushing 1588230740/info: appending metadata at 1732253744671 (+15 ms)Flushing 1588230740/info: closing flushed file at 1732253744671Flushing 1588230740/ns: creating writer at 1732253744684 (+13 ms)Flushing 1588230740/ns: appending metadata at 1732253744698 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732253744698Flushing 1588230740/table: creating writer at 1732253745115 (+417 ms)Flushing 1588230740/table: appending metadata at 1732253745130 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732253745131 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e23b2ae: reopening flushed file at 1732253745146 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bc195cb: reopening flushed file at 1732253745157 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d7bc856: reopening flushed file at 1732253745167 (+10 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 539ms, sequenceid=11, compaction requested=false at 1732253745177 (+10 ms)Writing region close event to WAL at 1732253745179 (+2 ms)Running coprocessor post-close hooks at 1732253745185 (+6 ms)Closed at 1732253745185
2024-11-22T05:35:45,186 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-22T05:35:45,216 INFO  [regionserver/82891a793a29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-22T05:35:45,217 INFO  [regionserver/82891a793a29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-22T05:35:45,222 INFO  [regionserver/82891a793a29:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T05:35:45,231 INFO  [regionserver/82891a793a29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-22T05:35:45,231 INFO  [regionserver/82891a793a29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-22T05:35:45,239 INFO  [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(976): stopping server 82891a793a29,35099,1732253704679; all regions closed.
2024-11-22T05:35:45,239 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:45,239 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:45,239 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:45,240 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:45,240 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:45,242 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741889_1074 (size=825)
2024-11-22T05:35:45,242 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741889_1074 (size=825)
2024-11-22T05:35:45,942 INFO  [master/82891a793a29:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-22T05:35:45,942 INFO  [master/82891a793a29:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-22T05:35:46,178 INFO  [regionserver/82891a793a29:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T05:35:46,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741836_1012 (size=76)
2024-11-22T05:35:46,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741832_1008 (size=32)
2024-11-22T05:35:47,220 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@70b64856[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42715, datanodeUuid=7022ae67-a4d8-42d1-b7a4-82fbd184c97e, infoPort=43729, infoSecurePort=0, ipcPort=36417, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022):Failed to transfer BP-860069939-172.17.0.2-1732253702022:blk_1073741826_1002 to 127.0.0.1:44683 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:47,220 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@382e1a59[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42715, datanodeUuid=7022ae67-a4d8-42d1-b7a4-82fbd184c97e, infoPort=43729, infoSecurePort=0, ipcPort=36417, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022):Failed to transfer BP-860069939-172.17.0.2-1732253702022:blk_1073741828_1004 to 127.0.0.1:44683 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:48,221 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@382e1a59[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42715, datanodeUuid=7022ae67-a4d8-42d1-b7a4-82fbd184c97e, infoPort=43729, infoSecurePort=0, ipcPort=36417, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022):Failed to transfer BP-860069939-172.17.0.2-1732253702022:blk_1073741829_1005 to 127.0.0.1:44683 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:48,221 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@70b64856[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42715, datanodeUuid=7022ae67-a4d8-42d1-b7a4-82fbd184c97e, infoPort=43729, infoSecurePort=0, ipcPort=36417, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022):Failed to transfer BP-860069939-172.17.0.2-1732253702022:blk_1073741827_1003 to 127.0.0.1:44683 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:48,286 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5fbd135[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36049, datanodeUuid=997782d0-6ecf-48b6-aea7-d2c60565a4a7, infoPort=33343, infoSecurePort=0, ipcPort=45673, storageInfo=lv=-57;cid=testClusterID;nsid=1702505774;c=1732253702022):Failed to transfer BP-860069939-172.17.0.2-1732253702022:blk_1073741861_1044 to 127.0.0.1:44683 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:48,646 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319 after 4002ms
2024-11-22T05:35:48,656 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta after 4001ms
2024-11-22T05:35:49,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741825_1001 (size=7)
2024-11-22T05:35:49,244 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@86441b8 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-860069939-172.17.0.2-1732253702022:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:41019,null,null])
java.net.ConnectException: Call From 82891a793a29/172.17.0.2 to localhost:34853 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-22T05:35:49,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-22T05:35:49,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T05:35:49,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-22T05:35:49,644 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-22T05:35:49,647 DEBUG [RS:1;82891a793a29:33487 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/oldWALs
2024-11-22T05:35:49,647 INFO  [RS:1;82891a793a29:33487 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 82891a793a29%2C33487%2C1732253706069:(num 1732253706319)
2024-11-22T05:35:49,647 DEBUG [RS:1;82891a793a29:33487 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:35:49,647 INFO  [RS:1;82891a793a29:33487 {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T05:35:49,647 INFO  [RS:1;82891a793a29:33487 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T05:35:49,647 INFO  [RS:1;82891a793a29:33487 {}] hbase.ChoreService(370): Chore service for: regionserver/82891a793a29:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-22T05:35:49,647 INFO  [RS:1;82891a793a29:33487 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-22T05:35:49,647 INFO  [RS:1;82891a793a29:33487 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-22T05:35:49,647 INFO  [RS:1;82891a793a29:33487 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-22T05:35:49,648 INFO  [RS:1;82891a793a29:33487 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T05:35:49,648 INFO  [RS:1;82891a793a29:33487 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33487
2024-11-22T05:35:49,648 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T05:35:49,653 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:49,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33487-0x1016109dcbc0002, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/82891a793a29,33487,1732253706069
2024-11-22T05:35:49,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T05:35:49,731 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [82891a793a29,33487,1732253706069]
2024-11-22T05:35:49,737 INFO  [RS:1;82891a793a29:33487 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T05:35:49,740 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/82891a793a29,33487,1732253706069 already deleted, retry=false
2024-11-22T05:35:49,740 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 82891a793a29,33487,1732253706069 expired; onlineServers=1
2024-11-22T05:35:49,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33487-0x1016109dcbc0002, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:35:49,851 INFO  [RS:1;82891a793a29:33487 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T05:35:49,851 INFO  [RS:1;82891a793a29:33487 {}] regionserver.HRegionServer(1031): Exiting; stopping=82891a793a29,33487,1732253706069; zookeeper connection closed.
2024-11-22T05:35:49,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33487-0x1016109dcbc0002, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:35:49,851 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7e3d5ba7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7e3d5ba7
2024-11-22T05:35:50,087 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:50,107 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:50,108 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:50,108 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:50,109 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:50,109 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:50,136 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:50,137 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:50,240 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-22T05:35:50,244 DEBUG [RS:0;82891a793a29:35099 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/oldWALs
2024-11-22T05:35:50,244 INFO  [RS:0;82891a793a29:35099 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 82891a793a29%2C35099%2C1732253704679.meta:.meta(num 1732253744639)
2024-11-22T05:35:50,245 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:50,245 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:50,245 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:50,245 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:50,245 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:50,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741880_1064 (size=18156)
2024-11-22T05:35:50,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741880_1064 (size=18156)
2024-11-22T05:35:50,254 DEBUG [RS:0;82891a793a29:35099 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/oldWALs
2024-11-22T05:35:50,254 INFO  [RS:0;82891a793a29:35099 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 82891a793a29%2C35099%2C1732253704679:(num 1732253743913)
2024-11-22T05:35:50,254 DEBUG [RS:0;82891a793a29:35099 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:35:50,254 INFO  [RS:0;82891a793a29:35099 {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T05:35:50,255 INFO  [RS:0;82891a793a29:35099 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T05:35:50,255 INFO  [RS:0;82891a793a29:35099 {}] hbase.ChoreService(370): Chore service for: regionserver/82891a793a29:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-22T05:35:50,255 INFO  [RS:0;82891a793a29:35099 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T05:35:50,255 INFO  [RS:0;82891a793a29:35099 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35099
2024-11-22T05:35:50,256 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T05:35:50,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/82891a793a29,35099,1732253704679
2024-11-22T05:35:50,308 INFO  [RS:0;82891a793a29:35099 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T05:35:50,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T05:35:50,319 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [82891a793a29,35099,1732253704679]
2024-11-22T05:35:50,329 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/82891a793a29,35099,1732253704679 already deleted, retry=false
2024-11-22T05:35:50,329 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 82891a793a29,35099,1732253704679 expired; onlineServers=0
2024-11-22T05:35:50,329 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '82891a793a29,36973,1732253704491' *****
2024-11-22T05:35:50,329 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-22T05:35:50,330 INFO  [M:0;82891a793a29:36973 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T05:35:50,330 INFO  [M:0;82891a793a29:36973 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T05:35:50,330 DEBUG [M:0;82891a793a29:36973 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-22T05:35:50,330 DEBUG [M:0;82891a793a29:36973 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-22T05:35:50,330 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-22T05:35:50,330 DEBUG [master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253705135 {}] cleaner.HFileCleaner(306): Exit Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253705135,5,FailOnTimeoutGroup]
2024-11-22T05:35:50,330 DEBUG [master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253705135 {}] cleaner.HFileCleaner(306): Exit Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253705135,5,FailOnTimeoutGroup]
2024-11-22T05:35:50,330 INFO  [M:0;82891a793a29:36973 {}] hbase.ChoreService(370): Chore service for: master/82891a793a29:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-22T05:35:50,330 INFO  [M:0;82891a793a29:36973 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T05:35:50,330 DEBUG [M:0;82891a793a29:36973 {}] master.HMaster(1795): Stopping service threads
2024-11-22T05:35:50,330 INFO  [M:0;82891a793a29:36973 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-22T05:35:50,330 INFO  [M:0;82891a793a29:36973 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T05:35:50,331 INFO  [M:0;82891a793a29:36973 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-22T05:35:50,331 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-22T05:35:50,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-22T05:35:50,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:50,340 DEBUG [M:0;82891a793a29:36973 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false
2024-11-22T05:35:50,340 DEBUG [M:0;82891a793a29:36973 {}] master.ActiveMasterManager(353): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master
2024-11-22T05:35:50,341 INFO  [M:0;82891a793a29:36973 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/.lastflushedseqids
2024-11-22T05:35:50,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741895_1081 (size=130)
2024-11-22T05:35:50,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741895_1081 (size=130)
2024-11-22T05:35:50,353 INFO  [M:0;82891a793a29:36973 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-22T05:35:50,353 INFO  [M:0;82891a793a29:36973 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-22T05:35:50,353 DEBUG [M:0;82891a793a29:36973 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T05:35:50,353 INFO  [M:0;82891a793a29:36973 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:35:50,353 DEBUG [M:0;82891a793a29:36973 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:35:50,353 DEBUG [M:0;82891a793a29:36973 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T05:35:50,353 DEBUG [M:0;82891a793a29:36973 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:35:50,354 INFO  [M:0;82891a793a29:36973 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB
2024-11-22T05:35:50,373 DEBUG [M:0;82891a793a29:36973 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/493596e5c826482ab5e3dfbbcf349247 is 82, key is hbase:meta,,1/info:regioninfo/1732253705872/Put/seqid=0
2024-11-22T05:35:50,383 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741896_1082 (size=5672)
2024-11-22T05:35:50,383 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741896_1082 (size=5672)
2024-11-22T05:35:50,384 INFO  [M:0;82891a793a29:36973 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/493596e5c826482ab5e3dfbbcf349247
2024-11-22T05:35:50,409 DEBUG [M:0;82891a793a29:36973 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/69761535b5e549bcbf94fd5b0792ea92 is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732253706622/Put/seqid=0
2024-11-22T05:35:50,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:35:50,419 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741897_1083 (size=6255)
2024-11-22T05:35:50,419 INFO  [RS:0;82891a793a29:35099 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T05:35:50,419 INFO  [RS:0;82891a793a29:35099 {}] regionserver.HRegionServer(1031): Exiting; stopping=82891a793a29,35099,1732253704679; zookeeper connection closed.
2024-11-22T05:35:50,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35099-0x1016109dcbc0001, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:35:50,419 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@647d68c9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@647d68c9
2024-11-22T05:35:50,419 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741897_1083 (size=6255)
2024-11-22T05:35:50,419 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-11-22T05:35:50,419 INFO  [M:0;82891a793a29:36973 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/69761535b5e549bcbf94fd5b0792ea92
2024-11-22T05:35:50,426 INFO  [M:0;82891a793a29:36973 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 69761535b5e549bcbf94fd5b0792ea92
2024-11-22T05:35:50,451 DEBUG [M:0;82891a793a29:36973 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1a401937eae14ecda870b5a0502a03c1 is 69, key is 82891a793a29,33487,1732253706069/rs:state/1732253706153/Put/seqid=0
2024-11-22T05:35:50,462 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741898_1084 (size=5224)
2024-11-22T05:35:50,463 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741898_1084 (size=5224)
2024-11-22T05:35:50,463 INFO  [M:0;82891a793a29:36973 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1a401937eae14ecda870b5a0502a03c1
2024-11-22T05:35:50,494 DEBUG [M:0;82891a793a29:36973 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/04091aac36cd4fec8e797394ddc887e0 is 52, key is load_balancer_on/state:d/1732253706051/Put/seqid=0
2024-11-22T05:35:50,499 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741899_1085 (size=5056)
2024-11-22T05:35:50,500 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741899_1085 (size=5056)
2024-11-22T05:35:50,500 INFO  [M:0;82891a793a29:36973 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/04091aac36cd4fec8e797394ddc887e0
2024-11-22T05:35:50,507 DEBUG [M:0;82891a793a29:36973 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/493596e5c826482ab5e3dfbbcf349247 as hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/493596e5c826482ab5e3dfbbcf349247
2024-11-22T05:35:50,513 INFO  [M:0;82891a793a29:36973 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/493596e5c826482ab5e3dfbbcf349247, entries=8, sequenceid=60, filesize=5.5 K
2024-11-22T05:35:50,514 DEBUG [M:0;82891a793a29:36973 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/69761535b5e549bcbf94fd5b0792ea92 as hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/69761535b5e549bcbf94fd5b0792ea92
2024-11-22T05:35:50,521 INFO  [M:0;82891a793a29:36973 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 69761535b5e549bcbf94fd5b0792ea92
2024-11-22T05:35:50,522 INFO  [M:0;82891a793a29:36973 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/69761535b5e549bcbf94fd5b0792ea92, entries=6, sequenceid=60, filesize=6.1 K
2024-11-22T05:35:50,523 DEBUG [M:0;82891a793a29:36973 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1a401937eae14ecda870b5a0502a03c1 as hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1a401937eae14ecda870b5a0502a03c1
2024-11-22T05:35:50,530 INFO  [M:0;82891a793a29:36973 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1a401937eae14ecda870b5a0502a03c1, entries=2, sequenceid=60, filesize=5.1 K
2024-11-22T05:35:50,531 DEBUG [M:0;82891a793a29:36973 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/04091aac36cd4fec8e797394ddc887e0 as hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/04091aac36cd4fec8e797394ddc887e0
2024-11-22T05:35:50,538 INFO  [M:0;82891a793a29:36973 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/04091aac36cd4fec8e797394ddc887e0, entries=1, sequenceid=60, filesize=4.9 K
2024-11-22T05:35:50,540 INFO  [M:0;82891a793a29:36973 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 187ms, sequenceid=60, compaction requested=false
2024-11-22T05:35:50,550 INFO  [M:0;82891a793a29:36973 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:35:50,550 DEBUG [M:0;82891a793a29:36973 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732253750353Disabling compacts and flushes for region at 1732253750353Disabling writes for close at 1732253750353Obtaining lock to block concurrent updates at 1732253750354 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732253750354Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1732253750354Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732253750355 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732253750355Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732253750372 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732253750372Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732253750392 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732253750408 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732253750408Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732253750426 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732253750451 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732253750451Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732253750470 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732253750493 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732253750493Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59e56cac: reopening flushed file at 1732253750506 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7daf51cf: reopening flushed file at 1732253750513 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79f52124: reopening flushed file at 1732253750522 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25df7f12: reopening flushed file at 1732253750530 (+8 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 187ms, sequenceid=60, compaction requested=false at 1732253750540 (+10 ms)Writing region close event to WAL at 1732253750549 (+9 ms)Closed at 1732253750549
2024-11-22T05:35:50,551 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:50,551 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:50,552 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:50,553 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:50,553 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:35:50,555 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36049 is added to blk_1073741878_1061 (size=1045)
2024-11-22T05:35:50,556 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741878_1061 (size=1045)
2024-11-22T05:35:50,557 INFO  [M:0;82891a793a29:36973 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-22T05:35:50,557 INFO  [M:0;82891a793a29:36973 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36973
2024-11-22T05:35:50,557 INFO  [M:0;82891a793a29:36973 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T05:35:50,557 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T05:35:50,638 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T05:35:50,655 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:50,656 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:50,657 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:50,657 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:50,658 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:50,658 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:50,657 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:50,658 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:50,664 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:50,667 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:35:50,672 INFO  [M:0;82891a793a29:36973 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T05:35:50,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:35:50,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1016109dcbc0000, quorum=127.0.0.1:59485, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:35:50,682 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@16711f95{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:35:50,682 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4595b5c3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:35:50,683 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:35:50,683 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@40e683d1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:35:50,683 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ecaf507{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.log.dir/,STOPPED}
2024-11-22T05:35:50,684 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@126de02d {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-860069939-172.17.0.2-1732253702022:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:41019,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:34853 , LocalHost:localPort 82891a793a29/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-22T05:35:50,685 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:35:50,685 WARN  [BP-860069939-172.17.0.2-1732253702022 heartbeating to localhost/127.0.0.1:46791 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:35:50,685 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:35:50,685 WARN  [BP-860069939-172.17.0.2-1732253702022 heartbeating to localhost/127.0.0.1:46791 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-860069939-172.17.0.2-1732253702022 (Datanode Uuid 7022ae67-a4d8-42d1-b7a4-82fbd184c97e) service to localhost/127.0.0.1:46791
2024-11-22T05:35:50,685 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@126de02d {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-860069939-172.17.0.2-1732253702022:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:41019,null,null], DatanodeInfoWithStorage[127.0.0.1:42715,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-860069939-172.17.0.2-1732253702022
2024-11-22T05:35:50,685 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@126de02d {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-860069939-172.17.0.2-1732253702022:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:42715,null,null])
java.io.IOException: No block pool offer service for bpid=BP-860069939-172.17.0.2-1732253702022
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:50,685 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@126de02d {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-860069939-172.17.0.2-1732253702022:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:41019,null,null])
java.io.IOException: No block pool offer service for bpid=BP-860069939-172.17.0.2-1732253702022
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:35:50,686 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@126de02d {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-860069939-172.17.0.2-1732253702022:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:42715,null,null], DatanodeInfoWithStorage[127.0.0.1:41019,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-860069939-172.17.0.2-1732253702022:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:42715,null,null], DatanodeInfoWithStorage[127.0.0.1:41019,null,null]]
2024-11-22T05:35:50,687 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data3/current/BP-860069939-172.17.0.2-1732253702022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:35:50,688 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data4/current/BP-860069939-172.17.0.2-1732253702022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:35:50,688 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:35:50,725 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@70a0068f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:35:50,726 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@39615ad8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:35:50,726 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:35:50,726 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5bea65f7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:35:50,726 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3eb2cdd3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.log.dir/,STOPPED}
2024-11-22T05:35:50,729 WARN  [BP-860069939-172.17.0.2-1732253702022 heartbeating to localhost/127.0.0.1:46791 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:35:50,729 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:35:50,729 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:35:50,729 WARN  [BP-860069939-172.17.0.2-1732253702022 heartbeating to localhost/127.0.0.1:46791 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-860069939-172.17.0.2-1732253702022 (Datanode Uuid 997782d0-6ecf-48b6-aea7-d2c60565a4a7) service to localhost/127.0.0.1:46791
2024-11-22T05:35:50,730 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data7/current/BP-860069939-172.17.0.2-1732253702022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:35:50,730 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/cluster_0389dc9c-be3b-e984-6e58-8ae5e097bca2/data/data8/current/BP-860069939-172.17.0.2-1732253702022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:35:50,730 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:35:50,739 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2295376c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T05:35:50,740 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@54adbc26{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:35:50,740 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:35:50,741 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4ac253d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:35:50,741 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ab5393f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.log.dir/,STOPPED}
2024-11-22T05:35:50,751 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-22T05:35:50,792 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-22T05:35:50,802 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=154 (was 79)
Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:46791
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46791
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:38997
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:46791
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46791 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-16-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46791 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-17-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46791
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:46791
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007fe0f4bf4db8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46791 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:38997
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46791 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46791
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46791
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007fe0f4bf4db8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=454 (was 402) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=246 (was 189) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7366 (was 8063)
2024-11-22T05:35:50,812 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=154, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=246, ProcessCount=11, AvailableMemoryMB=7366
2024-11-22T05:35:50,812 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-22T05:35:50,812 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.log.dir so I do NOT create it in target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6
2024-11-22T05:35:50,812 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6742f27e-8567-549c-e04f-37f00636c9ff/hadoop.tmp.dir so I do NOT create it in target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6
2024-11-22T05:35:50,813 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6, deleteOnExit=true
2024-11-22T05:35:50,813 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-22T05:35:50,813 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/test.cache.data in system properties and HBase conf
2024-11-22T05:35:50,813 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.tmp.dir in system properties and HBase conf
2024-11-22T05:35:50,813 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.log.dir in system properties and HBase conf
2024-11-22T05:35:50,813 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-22T05:35:50,813 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-22T05:35:50,813 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-22T05:35:50,814 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-22T05:35:50,814 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-22T05:35:50,814 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-22T05:35:50,814 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-22T05:35:50,814 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T05:35:50,814 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-22T05:35:50,814 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-22T05:35:50,814 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T05:35:50,814 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T05:35:50,814 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-22T05:35:50,815 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/nfs.dump.dir in system properties and HBase conf
2024-11-22T05:35:50,815 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/java.io.tmpdir in system properties and HBase conf
2024-11-22T05:35:50,815 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T05:35:50,815 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-22T05:35:50,815 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-22T05:35:50,832 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T05:35:51,172 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:35:51,180 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:35:51,189 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:35:51,189 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:35:51,190 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T05:35:51,190 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:35:51,191 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5493c38f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:35:51,191 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78e57521{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:35:51,325 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@42369482{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/java.io.tmpdir/jetty-localhost-44113-hadoop-hdfs-3_4_1-tests_jar-_-any-13868537256055622536/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T05:35:51,326 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@159bbbac{HTTP/1.1, (http/1.1)}{localhost:44113}
2024-11-22T05:35:51,326 INFO  [Time-limited test {}] server.Server(415): Started @158583ms
2024-11-22T05:35:51,343 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T05:35:51,655 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:51,658 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:51,965 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:35:51,970 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:35:51,973 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:35:51,973 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:35:51,973 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T05:35:51,977 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a6464d2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:35:51,977 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17be9294{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:35:52,094 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1fc8bed8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/java.io.tmpdir/jetty-localhost-41257-hadoop-hdfs-3_4_1-tests_jar-_-any-8826553583893096902/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:35:52,095 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7dee0203{HTTP/1.1, (http/1.1)}{localhost:41257}
2024-11-22T05:35:52,095 INFO  [Time-limited test {}] server.Server(415): Started @159353ms
2024-11-22T05:35:52,096 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:35:52,159 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:35:52,163 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:35:52,168 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:35:52,168 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:35:52,169 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:35:52,170 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34e43275{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:35:52,170 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16de61af{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:35:52,288 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@37b5496d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/java.io.tmpdir/jetty-localhost-46099-hadoop-hdfs-3_4_1-tests_jar-_-any-2015958888721568473/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:35:52,289 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@372c4e81{HTTP/1.1, (http/1.1)}{localhost:46099}
2024-11-22T05:35:52,289 INFO  [Time-limited test {}] server.Server(415): Started @159546ms
2024-11-22T05:35:52,290 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:35:52,656 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:52,659 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:53,114 WARN  [Thread-1212 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/data/data1/current/BP-2132726750-172.17.0.2-1732253750847/current, will proceed with Du for space computation calculation, 
2024-11-22T05:35:53,114 WARN  [Thread-1213 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/data/data2/current/BP-2132726750-172.17.0.2-1732253750847/current, will proceed with Du for space computation calculation, 
2024-11-22T05:35:53,128 WARN  [Thread-1176 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:35:53,130 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x972898903a982e03 with lease ID 0x2fe2e139ffd6cbba: Processing first storage report for DS-ba03f590-e634-4a24-9cd0-1da6a86808c4 from datanode DatanodeRegistration(127.0.0.1:43725, datanodeUuid=ef78f60e-06a3-423f-8d31-3d9ef5a1ab0f, infoPort=41295, infoSecurePort=0, ipcPort=38885, storageInfo=lv=-57;cid=testClusterID;nsid=649795978;c=1732253750847)
2024-11-22T05:35:53,131 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x972898903a982e03 with lease ID 0x2fe2e139ffd6cbba: from storage DS-ba03f590-e634-4a24-9cd0-1da6a86808c4 node DatanodeRegistration(127.0.0.1:43725, datanodeUuid=ef78f60e-06a3-423f-8d31-3d9ef5a1ab0f, infoPort=41295, infoSecurePort=0, ipcPort=38885, storageInfo=lv=-57;cid=testClusterID;nsid=649795978;c=1732253750847), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:35:53,131 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x972898903a982e03 with lease ID 0x2fe2e139ffd6cbba: Processing first storage report for DS-ae8e6145-0240-4bcc-9edc-c9d02319350f from datanode DatanodeRegistration(127.0.0.1:43725, datanodeUuid=ef78f60e-06a3-423f-8d31-3d9ef5a1ab0f, infoPort=41295, infoSecurePort=0, ipcPort=38885, storageInfo=lv=-57;cid=testClusterID;nsid=649795978;c=1732253750847)
2024-11-22T05:35:53,131 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x972898903a982e03 with lease ID 0x2fe2e139ffd6cbba: from storage DS-ae8e6145-0240-4bcc-9edc-c9d02319350f node DatanodeRegistration(127.0.0.1:43725, datanodeUuid=ef78f60e-06a3-423f-8d31-3d9ef5a1ab0f, infoPort=41295, infoSecurePort=0, ipcPort=38885, storageInfo=lv=-57;cid=testClusterID;nsid=649795978;c=1732253750847), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:35:53,320 WARN  [Thread-1223 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/data/data3/current/BP-2132726750-172.17.0.2-1732253750847/current, will proceed with Du for space computation calculation, 
2024-11-22T05:35:53,320 WARN  [Thread-1224 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/data/data4/current/BP-2132726750-172.17.0.2-1732253750847/current, will proceed with Du for space computation calculation, 
2024-11-22T05:35:53,334 WARN  [Thread-1199 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:35:53,336 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x79750bf7b5ebaef9 with lease ID 0x2fe2e139ffd6cbbb: Processing first storage report for DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc from datanode DatanodeRegistration(127.0.0.1:40329, datanodeUuid=e6df19ae-c2fa-4664-b025-3b09edab69c2, infoPort=36689, infoSecurePort=0, ipcPort=46073, storageInfo=lv=-57;cid=testClusterID;nsid=649795978;c=1732253750847)
2024-11-22T05:35:53,336 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x79750bf7b5ebaef9 with lease ID 0x2fe2e139ffd6cbbb: from storage DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc node DatanodeRegistration(127.0.0.1:40329, datanodeUuid=e6df19ae-c2fa-4664-b025-3b09edab69c2, infoPort=36689, infoSecurePort=0, ipcPort=46073, storageInfo=lv=-57;cid=testClusterID;nsid=649795978;c=1732253750847), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:35:53,336 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x79750bf7b5ebaef9 with lease ID 0x2fe2e139ffd6cbbb: Processing first storage report for DS-c3110d24-b5eb-4d87-b263-958a883db4db from datanode DatanodeRegistration(127.0.0.1:40329, datanodeUuid=e6df19ae-c2fa-4664-b025-3b09edab69c2, infoPort=36689, infoSecurePort=0, ipcPort=46073, storageInfo=lv=-57;cid=testClusterID;nsid=649795978;c=1732253750847)
2024-11-22T05:35:53,337 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x79750bf7b5ebaef9 with lease ID 0x2fe2e139ffd6cbbb: from storage DS-c3110d24-b5eb-4d87-b263-958a883db4db node DatanodeRegistration(127.0.0.1:40329, datanodeUuid=e6df19ae-c2fa-4664-b025-3b09edab69c2, infoPort=36689, infoSecurePort=0, ipcPort=46073, storageInfo=lv=-57;cid=testClusterID;nsid=649795978;c=1732253750847), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:35:53,340 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6
2024-11-22T05:35:53,343 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/zookeeper_0, clientPort=51421, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-22T05:35:53,343 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51421
2024-11-22T05:35:53,344 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:53,345 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:53,353 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40329 is added to blk_1073741825_1001 (size=7)
2024-11-22T05:35:53,353 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43725 is added to blk_1073741825_1001 (size=7)
2024-11-22T05:35:53,355 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4 with version=8
2024-11-22T05:35:53,355 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/hbase-staging
2024-11-22T05:35:53,357 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/82891a793a29:0 server-side Connection retries=45
2024-11-22T05:35:53,357 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:35:53,357 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T05:35:53,358 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T05:35:53,358 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:35:53,358 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T05:35:53,358 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-22T05:35:53,358 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T05:35:53,359 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37779
2024-11-22T05:35:53,361 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37779 connecting to ZooKeeper ensemble=127.0.0.1:51421
2024-11-22T05:35:53,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:377790x0, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T05:35:53,416 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37779-0x101610a9ba90000 connected
2024-11-22T05:35:53,502 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:53,504 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:53,506 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:35:53,506 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4, hbase.cluster.distributed=false
2024-11-22T05:35:53,508 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T05:35:53,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37779
2024-11-22T05:35:53,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37779
2024-11-22T05:35:53,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37779
2024-11-22T05:35:53,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37779
2024-11-22T05:35:53,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37779
2024-11-22T05:35:53,525 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/82891a793a29:0 server-side Connection retries=45
2024-11-22T05:35:53,525 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:35:53,525 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T05:35:53,525 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T05:35:53,525 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:35:53,525 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T05:35:53,525 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-22T05:35:53,526 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T05:35:53,526 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39785
2024-11-22T05:35:53,528 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39785 connecting to ZooKeeper ensemble=127.0.0.1:51421
2024-11-22T05:35:53,528 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:53,530 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:53,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:397850x0, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T05:35:53,544 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:397850x0, quorum=127.0.0.1:51421, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:35:53,544 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39785-0x101610a9ba90001 connected
2024-11-22T05:35:53,544 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-22T05:35:53,545 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-22T05:35:53,545 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-22T05:35:53,546 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T05:35:53,546 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39785
2024-11-22T05:35:53,547 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39785
2024-11-22T05:35:53,547 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39785
2024-11-22T05:35:53,547 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39785
2024-11-22T05:35:53,547 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39785
2024-11-22T05:35:53,561 DEBUG [M:0;82891a793a29:37779 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;82891a793a29:37779
2024-11-22T05:35:53,561 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/82891a793a29,37779,1732253753357
2024-11-22T05:35:53,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:35:53,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:35:53,575 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/82891a793a29,37779,1732253753357
2024-11-22T05:35:53,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:53,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-22T05:35:53,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:53,586 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-22T05:35:53,586 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/82891a793a29,37779,1732253753357 from backup master directory
2024-11-22T05:35:53,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:35:53,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/82891a793a29,37779,1732253753357
2024-11-22T05:35:53,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:35:53,596 WARN  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T05:35:53,596 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=82891a793a29,37779,1732253753357
2024-11-22T05:35:53,601 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/hbase.id] with ID: a8ad4116-d879-4e40-b31b-f73abe0223a0
2024-11-22T05:35:53,601 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/.tmp/hbase.id
2024-11-22T05:35:53,607 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43725 is added to blk_1073741826_1002 (size=42)
2024-11-22T05:35:53,608 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40329 is added to blk_1073741826_1002 (size=42)
2024-11-22T05:35:53,608 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/.tmp/hbase.id]:[hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/hbase.id]
2024-11-22T05:35:53,623 INFO  [master/82891a793a29:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:53,624 INFO  [master/82891a793a29:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-22T05:35:53,625 INFO  [master/82891a793a29:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-22T05:35:53,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:53,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:53,645 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40329 is added to blk_1073741827_1003 (size=196)
2024-11-22T05:35:53,645 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43725 is added to blk_1073741827_1003 (size=196)
2024-11-22T05:35:53,646 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T05:35:53,647 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-22T05:35:53,647 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:35:53,656 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40329 is added to blk_1073741828_1004 (size=1189)
2024-11-22T05:35:53,656 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43725 is added to blk_1073741828_1004 (size=1189)
2024-11-22T05:35:53,657 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store
2024-11-22T05:35:53,657 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:53,660 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:53,664 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40329 is added to blk_1073741829_1005 (size=34)
2024-11-22T05:35:53,664 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43725 is added to blk_1073741829_1005 (size=34)
2024-11-22T05:35:53,665 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:35:53,665 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T05:35:53,665 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:35:53,665 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:35:53,665 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T05:35:53,665 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:35:53,665 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:35:53,665 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732253753665Disabling compacts and flushes for region at 1732253753665Disabling writes for close at 1732253753665Writing region close event to WAL at 1732253753665Closed at 1732253753665
2024-11-22T05:35:53,666 WARN  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/.initializing
2024-11-22T05:35:53,666 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/WALs/82891a793a29,37779,1732253753357
2024-11-22T05:35:53,673 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C37779%2C1732253753357, suffix=, logDir=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/WALs/82891a793a29,37779,1732253753357, archiveDir=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/oldWALs, maxLogs=10
2024-11-22T05:35:53,673 INFO  [master/82891a793a29:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C37779%2C1732253753357.1732253753673
2024-11-22T05:35:53,678 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/WALs/82891a793a29,37779,1732253753357/82891a793a29%2C37779%2C1732253753357.1732253753673
2024-11-22T05:35:53,679 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36689:36689),(127.0.0.1/127.0.0.1:41295:41295)]
2024-11-22T05:35:53,680 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:35:53,680 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:35:53,680 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:35:53,680 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:35:53,684 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:35:53,686 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-22T05:35:53,686 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:53,687 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:53,687 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:35:53,688 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-22T05:35:53,688 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:53,689 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:35:53,689 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:35:53,690 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-22T05:35:53,691 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:53,691 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:35:53,691 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:35:53,693 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-22T05:35:53,693 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:53,693 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:35:53,693 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:35:53,694 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:35:53,694 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:35:53,696 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:35:53,696 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:35:53,696 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-22T05:35:53,697 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:35:53,701 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:35:53,701 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=872121, jitterRate=0.10895930230617523}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-22T05:35:53,702 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732253753680Initializing all the Stores at 1732253753681 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253753681Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253753684 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253753684Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253753684Cleaning up temporary data from old regions at 1732253753696 (+12 ms)Region opened successfully at 1732253753702 (+6 ms)
2024-11-22T05:35:53,702 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-22T05:35:53,705 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ad83416, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82891a793a29/172.17.0.2:0
2024-11-22T05:35:53,706 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-22T05:35:53,707 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-22T05:35:53,707 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-22T05:35:53,707 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-22T05:35:53,707 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-22T05:35:53,708 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-22T05:35:53,708 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-22T05:35:53,710 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-22T05:35:53,711 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-22T05:35:53,722 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-22T05:35:53,723 INFO  [master/82891a793a29:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-22T05:35:53,724 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-22T05:35:53,733 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-22T05:35:53,733 INFO  [master/82891a793a29:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-22T05:35:53,734 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-22T05:35:53,744 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-22T05:35:53,745 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-22T05:35:53,754 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-22T05:35:53,756 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-22T05:35:53,764 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-22T05:35:53,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T05:35:53,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:53,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T05:35:53,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:53,776 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=82891a793a29,37779,1732253753357, sessionid=0x101610a9ba90000, setting cluster-up flag (Was=false)
2024-11-22T05:35:53,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:53,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:53,828 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-22T05:35:53,829 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=82891a793a29,37779,1732253753357
2024-11-22T05:35:54,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:54,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:54,175 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-22T05:35:54,176 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=82891a793a29,37779,1732253753357
2024-11-22T05:35:54,178 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-22T05:35:54,180 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-22T05:35:54,180 INFO  [master/82891a793a29:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-22T05:35:54,180 INFO  [master/82891a793a29:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-22T05:35:54,181 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 82891a793a29,37779,1732253753357
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-22T05:35:54,182 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:35:54,182 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:35:54,182 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:35:54,183 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:35:54,183 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/82891a793a29:0, corePoolSize=10, maxPoolSize=10
2024-11-22T05:35:54,183 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:54,183 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/82891a793a29:0, corePoolSize=2, maxPoolSize=2
2024-11-22T05:35:54,183 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:54,185 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:35:54,186 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-22T05:35:54,187 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:54,187 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-22T05:35:54,199 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732253784199
2024-11-22T05:35:54,199 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-22T05:35:54,200 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-22T05:35:54,200 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-22T05:35:54,200 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-22T05:35:54,200 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-22T05:35:54,200 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-22T05:35:54,200 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,200 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-22T05:35:54,200 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-22T05:35:54,201 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-22T05:35:54,201 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-22T05:35:54,201 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-22T05:35:54,201 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253754201,5,FailOnTimeoutGroup]
2024-11-22T05:35:54,201 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253754201,5,FailOnTimeoutGroup]
2024-11-22T05:35:54,201 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,201 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-22T05:35:54,201 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,202 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,202 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43725 is added to blk_1073741831_1007 (size=1321)
2024-11-22T05:35:54,202 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40329 is added to blk_1073741831_1007 (size=1321)
2024-11-22T05:35:54,203 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-22T05:35:54,203 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4
2024-11-22T05:35:54,209 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40329 is added to blk_1073741832_1008 (size=32)
2024-11-22T05:35:54,209 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43725 is added to blk_1073741832_1008 (size=32)
2024-11-22T05:35:54,210 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:35:54,211 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T05:35:54,213 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T05:35:54,213 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:54,213 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:54,213 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T05:35:54,215 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T05:35:54,215 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:54,215 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:54,215 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T05:35:54,217 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T05:35:54,217 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:54,217 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:54,217 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T05:35:54,219 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T05:35:54,219 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:54,219 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:54,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T05:35:54,220 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740
2024-11-22T05:35:54,220 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740
2024-11-22T05:35:54,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T05:35:54,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T05:35:54,223 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T05:35:54,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T05:35:54,226 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:35:54,227 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=820593, jitterRate=0.043438106775283813}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T05:35:54,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732253754210Initializing all the Stores at 1732253754211 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253754211Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253754211Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253754211Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253754211Cleaning up temporary data from old regions at 1732253754222 (+11 ms)Region opened successfully at 1732253754228 (+6 ms)
2024-11-22T05:35:54,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T05:35:54,228 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T05:35:54,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T05:35:54,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T05:35:54,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T05:35:54,228 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T05:35:54,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732253754228Disabling compacts and flushes for region at 1732253754228Disabling writes for close at 1732253754228Writing region close event to WAL at 1732253754228Closed at 1732253754228
2024-11-22T05:35:54,230 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:35:54,230 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-22T05:35:54,230 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-22T05:35:54,231 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T05:35:54,232 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-22T05:35:54,249 INFO  [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer(746): ClusterId : a8ad4116-d879-4e40-b31b-f73abe0223a0
2024-11-22T05:35:54,249 DEBUG [RS:0;82891a793a29:39785 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-22T05:35:54,260 DEBUG [RS:0;82891a793a29:39785 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-22T05:35:54,260 DEBUG [RS:0;82891a793a29:39785 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-22T05:35:54,270 DEBUG [RS:0;82891a793a29:39785 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-22T05:35:54,271 DEBUG [RS:0;82891a793a29:39785 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63126704, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82891a793a29/172.17.0.2:0
2024-11-22T05:35:54,286 DEBUG [RS:0;82891a793a29:39785 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;82891a793a29:39785
2024-11-22T05:35:54,286 INFO  [RS:0;82891a793a29:39785 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-22T05:35:54,286 INFO  [RS:0;82891a793a29:39785 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-22T05:35:54,286 DEBUG [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-22T05:35:54,287 INFO  [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer(2659): reportForDuty to master=82891a793a29,37779,1732253753357 with port=39785, startcode=1732253753525
2024-11-22T05:35:54,287 DEBUG [RS:0;82891a793a29:39785 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-22T05:35:54,289 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54071, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-11-22T05:35:54,290 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37779 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 82891a793a29,39785,1732253753525
2024-11-22T05:35:54,290 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37779 {}] master.ServerManager(517): Registering regionserver=82891a793a29,39785,1732253753525
2024-11-22T05:35:54,292 DEBUG [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4
2024-11-22T05:35:54,292 DEBUG [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33625
2024-11-22T05:35:54,292 DEBUG [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-22T05:35:54,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T05:35:54,302 DEBUG [RS:0;82891a793a29:39785 {}] zookeeper.ZKUtil(111): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/82891a793a29,39785,1732253753525
2024-11-22T05:35:54,302 WARN  [RS:0;82891a793a29:39785 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T05:35:54,302 INFO  [RS:0;82891a793a29:39785 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:35:54,302 DEBUG [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525
2024-11-22T05:35:54,302 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [82891a793a29,39785,1732253753525]
2024-11-22T05:35:54,305 INFO  [RS:0;82891a793a29:39785 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-22T05:35:54,307 INFO  [RS:0;82891a793a29:39785 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-22T05:35:54,307 INFO  [RS:0;82891a793a29:39785 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T05:35:54,307 INFO  [RS:0;82891a793a29:39785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,308 INFO  [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-22T05:35:54,309 INFO  [RS:0;82891a793a29:39785 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-22T05:35:54,309 INFO  [RS:0;82891a793a29:39785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,309 DEBUG [RS:0;82891a793a29:39785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:54,309 DEBUG [RS:0;82891a793a29:39785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:54,309 DEBUG [RS:0;82891a793a29:39785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:54,309 DEBUG [RS:0;82891a793a29:39785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:54,309 DEBUG [RS:0;82891a793a29:39785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:54,309 DEBUG [RS:0;82891a793a29:39785 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/82891a793a29:0, corePoolSize=2, maxPoolSize=2
2024-11-22T05:35:54,309 DEBUG [RS:0;82891a793a29:39785 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:54,309 DEBUG [RS:0;82891a793a29:39785 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:54,309 DEBUG [RS:0;82891a793a29:39785 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:54,309 DEBUG [RS:0;82891a793a29:39785 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:54,309 DEBUG [RS:0;82891a793a29:39785 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:54,309 DEBUG [RS:0;82891a793a29:39785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:35:54,309 DEBUG [RS:0;82891a793a29:39785 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/82891a793a29:0, corePoolSize=3, maxPoolSize=3
2024-11-22T05:35:54,310 DEBUG [RS:0;82891a793a29:39785 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0, corePoolSize=3, maxPoolSize=3
2024-11-22T05:35:54,310 INFO  [RS:0;82891a793a29:39785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,310 INFO  [RS:0;82891a793a29:39785 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,310 INFO  [RS:0;82891a793a29:39785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,310 INFO  [RS:0;82891a793a29:39785 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,310 INFO  [RS:0;82891a793a29:39785 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,310 INFO  [RS:0;82891a793a29:39785 {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,39785,1732253753525-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T05:35:54,327 INFO  [RS:0;82891a793a29:39785 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-22T05:35:54,327 INFO  [RS:0;82891a793a29:39785 {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,39785,1732253753525-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,327 INFO  [RS:0;82891a793a29:39785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,327 INFO  [RS:0;82891a793a29:39785 {}] regionserver.Replication(171): 82891a793a29,39785,1732253753525 started
2024-11-22T05:35:54,342 INFO  [RS:0;82891a793a29:39785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,342 INFO  [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer(1482): Serving as 82891a793a29,39785,1732253753525, RpcServer on 82891a793a29/172.17.0.2:39785, sessionid=0x101610a9ba90001
2024-11-22T05:35:54,342 DEBUG [RS:0;82891a793a29:39785 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-22T05:35:54,342 DEBUG [RS:0;82891a793a29:39785 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 82891a793a29,39785,1732253753525
2024-11-22T05:35:54,342 DEBUG [RS:0;82891a793a29:39785 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82891a793a29,39785,1732253753525'
2024-11-22T05:35:54,342 DEBUG [RS:0;82891a793a29:39785 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-22T05:35:54,343 DEBUG [RS:0;82891a793a29:39785 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-22T05:35:54,343 DEBUG [RS:0;82891a793a29:39785 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-22T05:35:54,343 DEBUG [RS:0;82891a793a29:39785 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-22T05:35:54,343 DEBUG [RS:0;82891a793a29:39785 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 82891a793a29,39785,1732253753525
2024-11-22T05:35:54,343 DEBUG [RS:0;82891a793a29:39785 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82891a793a29,39785,1732253753525'
2024-11-22T05:35:54,343 DEBUG [RS:0;82891a793a29:39785 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-22T05:35:54,344 DEBUG [RS:0;82891a793a29:39785 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-22T05:35:54,344 DEBUG [RS:0;82891a793a29:39785 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-22T05:35:54,344 INFO  [RS:0;82891a793a29:39785 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-22T05:35:54,344 INFO  [RS:0;82891a793a29:39785 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-22T05:35:54,383 WARN  [82891a793a29:37779 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-22T05:35:54,447 INFO  [RS:0;82891a793a29:39785 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C39785%2C1732253753525, suffix=, logDir=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525, archiveDir=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/oldWALs, maxLogs=32
2024-11-22T05:35:54,448 INFO  [RS:0;82891a793a29:39785 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C39785%2C1732253753525.1732253754448
2024-11-22T05:35:54,461 INFO  [RS:0;82891a793a29:39785 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253754448
2024-11-22T05:35:54,462 DEBUG [RS:0;82891a793a29:39785 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36689:36689),(127.0.0.1/127.0.0.1:41295:41295)]
2024-11-22T05:35:54,633 DEBUG [82891a793a29:37779 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-22T05:35:54,634 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=82891a793a29,39785,1732253753525
2024-11-22T05:35:54,635 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 82891a793a29,39785,1732253753525, state=OPENING
2024-11-22T05:35:54,658 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:54,659 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-22T05:35:54,660 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:54,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:54,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:35:54,670 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T05:35:54,670 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=82891a793a29,39785,1732253753525}]
2024-11-22T05:35:54,670 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:35:54,670 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:35:54,824 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-22T05:35:54,826 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57283, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-22T05:35:54,830 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-22T05:35:54,830 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:35:54,832 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C39785%2C1732253753525.meta, suffix=.meta, logDir=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525, archiveDir=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/oldWALs, maxLogs=32
2024-11-22T05:35:54,833 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C39785%2C1732253753525.meta.1732253754832.meta
2024-11-22T05:35:54,841 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.meta.1732253754832.meta
2024-11-22T05:35:54,842 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41295:41295),(127.0.0.1/127.0.0.1:36689:36689)]
2024-11-22T05:35:54,842 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:35:54,843 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-22T05:35:54,843 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-22T05:35:54,843 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-22T05:35:54,843 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-22T05:35:54,843 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:35:54,843 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-22T05:35:54,843 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-22T05:35:54,845 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T05:35:54,846 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T05:35:54,846 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:54,846 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:54,846 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T05:35:54,847 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T05:35:54,847 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:54,848 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:54,848 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T05:35:54,849 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T05:35:54,849 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:54,849 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:54,849 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T05:35:54,850 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T05:35:54,850 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:54,850 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:35:54,851 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T05:35:54,851 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740
2024-11-22T05:35:54,852 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740
2024-11-22T05:35:54,854 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T05:35:54,854 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T05:35:54,854 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T05:35:54,856 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T05:35:54,857 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=756806, jitterRate=-0.03767186403274536}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T05:35:54,857 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-22T05:35:54,858 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732253754843Writing region info on filesystem at 1732253754843Initializing all the Stores at 1732253754844 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253754844Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253754845 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253754845Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253754845Cleaning up temporary data from old regions at 1732253754854 (+9 ms)Running coprocessor post-open hooks at 1732253754857 (+3 ms)Region opened successfully at 1732253754858 (+1 ms)
2024-11-22T05:35:54,859 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732253754824
2024-11-22T05:35:54,861 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-22T05:35:54,861 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-22T05:35:54,862 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=82891a793a29,39785,1732253753525
2024-11-22T05:35:54,863 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 82891a793a29,39785,1732253753525, state=OPEN
2024-11-22T05:35:54,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T05:35:54,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T05:35:54,928 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=82891a793a29,39785,1732253753525
2024-11-22T05:35:54,928 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:35:54,928 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:35:54,934 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-22T05:35:54,934 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=82891a793a29,39785,1732253753525 in 258 msec
2024-11-22T05:35:54,938 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-22T05:35:54,938 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 704 msec
2024-11-22T05:35:54,939 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:35:54,939 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-22T05:35:54,940 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T05:35:54,940 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82891a793a29,39785,1732253753525, seqNum=-1]
2024-11-22T05:35:54,940 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T05:35:54,942 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48641, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T05:35:54,948 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 768 msec
2024-11-22T05:35:54,948 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732253754948, completionTime=-1
2024-11-22T05:35:54,948 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-22T05:35:54,948 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-22T05:35:54,950 INFO  [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-22T05:35:54,950 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732253814950
2024-11-22T05:35:54,950 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732253874950
2024-11-22T05:35:54,950 INFO  [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-11-22T05:35:54,950 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,37779,1732253753357-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,950 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,37779,1732253753357-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,951 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,37779,1732253753357-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,951 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-82891a793a29:37779, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,951 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,951 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:54,953 DEBUG [master/82891a793a29:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-22T05:35:54,954 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.358sec
2024-11-22T05:35:54,955 INFO  [master/82891a793a29:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-22T05:35:54,955 INFO  [master/82891a793a29:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-22T05:35:54,955 INFO  [master/82891a793a29:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-22T05:35:54,955 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-22T05:35:54,955 INFO  [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-22T05:35:54,955 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,37779,1732253753357-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T05:35:54,955 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,37779,1732253753357-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-22T05:35:54,957 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-22T05:35:54,957 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-22T05:35:54,957 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,37779,1732253753357-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:35:55,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47828a5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:35:55,050 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82891a793a29,37779,-1 for getting cluster id
2024-11-22T05:35:55,050 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-22T05:35:55,052 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ad4116-d879-4e40-b31b-f73abe0223a0'
2024-11-22T05:35:55,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-22T05:35:55,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ad4116-d879-4e40-b31b-f73abe0223a0"

2024-11-22T05:35:55,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@611b1dfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:35:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82891a793a29,37779,-1]
2024-11-22T05:35:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-22T05:35:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:35:55,054 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-22T05:35:55,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@531497d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:35:55,056 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T05:35:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82891a793a29,39785,1732253753525, seqNum=-1]
2024-11-22T05:35:55,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T05:35:55,058 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T05:35:55,060 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=82891a793a29,37779,1732253753357
2024-11-22T05:35:55,060 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:35:55,063 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-22T05:35:55,063 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-11-22T05:35:55,063 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-11-22T05:35:55,064 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-22T05:35:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is 82891a793a29,37779,1732253753357
2024-11-22T05:35:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@668c3442
2024-11-22T05:35:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-22T05:35:55,071 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-22T05:35:55,071 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37779 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-22T05:35:55,072 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37779 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-22T05:35:55,072 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37779 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T05:35:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37779 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-11-22T05:35:55,075 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-11-22T05:35:55,075 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:55,075 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37779 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-11-22T05:35:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37779 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T05:35:55,077 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-22T05:35:55,088 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40329 is added to blk_1073741835_1011 (size=395)
2024-11-22T05:35:55,088 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43725 is added to blk_1073741835_1011 (size=395)
2024-11-22T05:35:55,090 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8e4107d9b30a70518c33762ff0d1bbc7, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4
2024-11-22T05:35:55,096 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40329 is added to blk_1073741836_1012 (size=78)
2024-11-22T05:35:55,097 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:35:55,097 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 8e4107d9b30a70518c33762ff0d1bbc7, disabling compactions & flushes
2024-11-22T05:35:55,097 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.
2024-11-22T05:35:55,097 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.
2024-11-22T05:35:55,097 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7. after waiting 0 ms
2024-11-22T05:35:55,097 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43725 is added to blk_1073741836_1012 (size=78)
2024-11-22T05:35:55,097 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.
2024-11-22T05:35:55,097 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.
2024-11-22T05:35:55,097 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8e4107d9b30a70518c33762ff0d1bbc7:
Waiting for close lock at 1732253755097Disabling compacts and flushes for region at 1732253755097Disabling writes for close at 1732253755097Writing region close event to WAL at 1732253755097Closed at 1732253755097
2024-11-22T05:35:55,099 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-11-22T05:35:55,099 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1732253755099"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732253755099"}]},"ts":"1732253755099"}
2024-11-22T05:35:55,102 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-22T05:35:55,103 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-22T05:35:55,104 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732253755103"}]},"ts":"1732253755103"}
2024-11-22T05:35:55,106 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-11-22T05:35:55,106 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=8e4107d9b30a70518c33762ff0d1bbc7, ASSIGN}]
2024-11-22T05:35:55,108 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=8e4107d9b30a70518c33762ff0d1bbc7, ASSIGN
2024-11-22T05:35:55,109 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=8e4107d9b30a70518c33762ff0d1bbc7, ASSIGN; state=OFFLINE, location=82891a793a29,39785,1732253753525; forceNewPlan=false, retain=false
2024-11-22T05:35:55,260 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8e4107d9b30a70518c33762ff0d1bbc7, regionState=OPENING, regionLocation=82891a793a29,39785,1732253753525
2024-11-22T05:35:55,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=8e4107d9b30a70518c33762ff0d1bbc7, ASSIGN because future has completed
2024-11-22T05:35:55,263 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e4107d9b30a70518c33762ff0d1bbc7, server=82891a793a29,39785,1732253753525}]
2024-11-22T05:35:55,421 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.
2024-11-22T05:35:55,421 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e4107d9b30a70518c33762ff0d1bbc7, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:35:55,421 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 8e4107d9b30a70518c33762ff0d1bbc7
2024-11-22T05:35:55,421 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:35:55,422 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8e4107d9b30a70518c33762ff0d1bbc7
2024-11-22T05:35:55,422 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8e4107d9b30a70518c33762ff0d1bbc7
2024-11-22T05:35:55,423 INFO  [StoreOpener-8e4107d9b30a70518c33762ff0d1bbc7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8e4107d9b30a70518c33762ff0d1bbc7 
2024-11-22T05:35:55,424 INFO  [StoreOpener-8e4107d9b30a70518c33762ff0d1bbc7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e4107d9b30a70518c33762ff0d1bbc7 columnFamilyName info
2024-11-22T05:35:55,424 DEBUG [StoreOpener-8e4107d9b30a70518c33762ff0d1bbc7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:35:55,425 INFO  [StoreOpener-8e4107d9b30a70518c33762ff0d1bbc7-1 {}] regionserver.HStore(327): Store=8e4107d9b30a70518c33762ff0d1bbc7/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:35:55,425 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8e4107d9b30a70518c33762ff0d1bbc7
2024-11-22T05:35:55,426 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/default/TestLogRolling-testLogRollOnPipelineRestart/8e4107d9b30a70518c33762ff0d1bbc7
2024-11-22T05:35:55,426 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/default/TestLogRolling-testLogRollOnPipelineRestart/8e4107d9b30a70518c33762ff0d1bbc7
2024-11-22T05:35:55,427 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8e4107d9b30a70518c33762ff0d1bbc7
2024-11-22T05:35:55,427 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8e4107d9b30a70518c33762ff0d1bbc7
2024-11-22T05:35:55,428 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8e4107d9b30a70518c33762ff0d1bbc7
2024-11-22T05:35:55,431 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/default/TestLogRolling-testLogRollOnPipelineRestart/8e4107d9b30a70518c33762ff0d1bbc7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:35:55,431 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8e4107d9b30a70518c33762ff0d1bbc7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=836812, jitterRate=0.06406164169311523}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-22T05:35:55,431 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e4107d9b30a70518c33762ff0d1bbc7
2024-11-22T05:35:55,432 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8e4107d9b30a70518c33762ff0d1bbc7:
Running coprocessor pre-open hook at 1732253755422Writing region info on filesystem at 1732253755422Initializing all the Stores at 1732253755422Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253755422Cleaning up temporary data from old regions at 1732253755427 (+5 ms)Running coprocessor post-open hooks at 1732253755431 (+4 ms)Region opened successfully at 1732253755432 (+1 ms)
2024-11-22T05:35:55,433 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7., pid=6, masterSystemTime=1732253755417
2024-11-22T05:35:55,436 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.
2024-11-22T05:35:55,436 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.
2024-11-22T05:35:55,437 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8e4107d9b30a70518c33762ff0d1bbc7, regionState=OPEN, openSeqNum=2, regionLocation=82891a793a29,39785,1732253753525
2024-11-22T05:35:55,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e4107d9b30a70518c33762ff0d1bbc7, server=82891a793a29,39785,1732253753525 because future has completed
2024-11-22T05:35:55,444 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-22T05:35:55,444 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8e4107d9b30a70518c33762ff0d1bbc7, server=82891a793a29,39785,1732253753525 in 178 msec
2024-11-22T05:35:55,447 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-22T05:35:55,447 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=8e4107d9b30a70518c33762ff0d1bbc7, ASSIGN in 338 msec
2024-11-22T05:35:55,448 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-22T05:35:55,449 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732253755448"}]},"ts":"1732253755448"}
2024-11-22T05:35:55,451 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-11-22T05:35:55,453 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-11-22T05:35:55,455 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 381 msec
2024-11-22T05:35:55,659 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:55,661 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:56,659 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:56,662 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:57,660 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:57,662 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:58,661 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:58,663 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:59,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-22T05:35:59,491 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-22T05:35:59,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-22T05:35:59,492 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-11-22T05:35:59,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T05:35:59,493 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-22T05:35:59,662 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:35:59,663 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:00,347 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T05:36:00,372 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:00,372 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:00,372 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:00,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:00,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:00,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:00,378 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:00,379 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:00,379 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:00,382 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:00,389 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-22T05:36:00,390 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-11-22T05:36:00,662 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:00,664 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:01,663 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:01,664 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:02,664 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:02,665 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:03,665 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:03,665 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:04,665 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:04,666 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37779 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T05:36:05,173 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-11-22T05:36:05,173 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-11-22T05:36:05,177 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-11-22T05:36:05,177 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.
2024-11-22T05:36:05,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7., hostname=82891a793a29,39785,1732253753525, seqNum=2]
2024-11-22T05:36:05,666 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:05,667 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:06,667 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:06,668 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:07,185 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253754448
2024-11-22T05:36:07,186 WARN  [ResponseProcessor for block BP-2132726750-172.17.0.2-1732253750847:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2132726750-172.17.0.2-1732253750847:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:07,186 WARN  [ResponseProcessor for block BP-2132726750-172.17.0.2-1732253750847:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2132726750-172.17.0.2-1732253750847:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-2132726750-172.17.0.2-1732253750847:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:40329,DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:07,186 WARN  [DataStreamer for file /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/WALs/82891a793a29,37779,1732253753357/82891a793a29%2C37779%2C1732253753357.1732253753673 block BP-2132726750-172.17.0.2-1732253750847:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2132726750-172.17.0.2-1732253750847:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40329,DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc,DISK], DatanodeInfoWithStorage[127.0.0.1:43725,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40329,DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc,DISK]) is bad.
2024-11-22T05:36:07,186 WARN  [ResponseProcessor for block BP-2132726750-172.17.0.2-1732253750847:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2132726750-172.17.0.2-1732253750847:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:07,187 WARN  [DataStreamer for file /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253754448 block BP-2132726750-172.17.0.2-1732253750847:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2132726750-172.17.0.2-1732253750847:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40329,DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc,DISK], DatanodeInfoWithStorage[127.0.0.1:43725,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40329,DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc,DISK]) is bad.
2024-11-22T05:36:07,187 WARN  [DataStreamer for file /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.meta.1732253754832.meta block BP-2132726750-172.17.0.2-1732253750847:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2132726750-172.17.0.2-1732253750847:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43725,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK], DatanodeInfoWithStorage[127.0.0.1:40329,DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40329,DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc,DISK]) is bad.
2024-11-22T05:36:07,187 WARN  [PacketResponder: BP-2132726750-172.17.0.2-1732253750847:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:40329] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:07,187 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_660166755_22 at /127.0.0.1:56036 [Receiving block BP-2132726750-172.17.0.2-1732253750847:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:40329:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56036 dst: /127.0.0.1:40329
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:07,188 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_660166755_22 at /127.0.0.1:51228 [Receiving block BP-2132726750-172.17.0.2-1732253750847:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:43725:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51228 dst: /127.0.0.1:43725
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:07,188 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_70066984_22 at /127.0.0.1:51268 [Receiving block BP-2132726750-172.17.0.2-1732253750847:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:43725:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51268 dst: /127.0.0.1:43725
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:07,188 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_70066984_22 at /127.0.0.1:56074 [Receiving block BP-2132726750-172.17.0.2-1732253750847:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:40329:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56074 dst: /127.0.0.1:40329
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:07,188 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_70066984_22 at /127.0.0.1:56082 [Receiving block BP-2132726750-172.17.0.2-1732253750847:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:40329:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56082 dst: /127.0.0.1:40329
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:07,188 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_70066984_22 at /127.0.0.1:51258 [Receiving block BP-2132726750-172.17.0.2-1732253750847:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:43725:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51258 dst: /127.0.0.1:43725
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:07,336 WARN  [BP-2132726750-172.17.0.2-1732253750847 heartbeating to localhost/127.0.0.1:33625 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2132726750-172.17.0.2-1732253750847 (Datanode Uuid e6df19ae-c2fa-4664-b025-3b09edab69c2) service to localhost/127.0.0.1:33625
2024-11-22T05:36:07,412 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/data/data3/current/BP-2132726750-172.17.0.2-1732253750847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:36:07,412 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/data/data4/current/BP-2132726750-172.17.0.2-1732253750847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:36:07,413 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@37b5496d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:36:07,414 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@372c4e81{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:36:07,414 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:36:07,414 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16de61af{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:36:07,414 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34e43275{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.log.dir/,STOPPED}
2024-11-22T05:36:07,416 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:36:07,426 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:36:07,430 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:36:07,431 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:36:07,431 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:36:07,431 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T05:36:07,432 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70770496{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:36:07,432 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48dc7e0c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:36:07,541 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2460467f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/java.io.tmpdir/jetty-localhost-42681-hadoop-hdfs-3_4_1-tests_jar-_-any-3159759317211326105/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:36:07,542 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a3c48a5{HTTP/1.1, (http/1.1)}{localhost:42681}
2024-11-22T05:36:07,542 INFO  [Time-limited test {}] server.Server(415): Started @174799ms
2024-11-22T05:36:07,543 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:36:07,572 WARN  [ResponseProcessor for block BP-2132726750-172.17.0.2-1732253750847:blk_1073741834_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2132726750-172.17.0.2-1732253750847:blk_1073741834_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:07,572 WARN  [ResponseProcessor for block BP-2132726750-172.17.0.2-1732253750847:blk_1073741833_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2132726750-172.17.0.2-1732253750847:blk_1073741833_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:07,572 WARN  [ResponseProcessor for block BP-2132726750-172.17.0.2-1732253750847:blk_1073741830_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2132726750-172.17.0.2-1732253750847:blk_1073741830_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:07,573 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_70066984_22 at /127.0.0.1:48658 [Receiving block BP-2132726750-172.17.0.2-1732253750847:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:43725:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48658 dst: /127.0.0.1:43725
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:07,574 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_660166755_22 at /127.0.0.1:48650 [Receiving block BP-2132726750-172.17.0.2-1732253750847:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:43725:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48650 dst: /127.0.0.1:43725
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:07,574 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_70066984_22 at /127.0.0.1:48656 [Receiving block BP-2132726750-172.17.0.2-1732253750847:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:43725:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48656 dst: /127.0.0.1:43725
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:07,576 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1fc8bed8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:36:07,576 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7dee0203{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:36:07,577 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:36:07,577 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17be9294{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:36:07,577 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a6464d2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.log.dir/,STOPPED}
2024-11-22T05:36:07,578 WARN  [BP-2132726750-172.17.0.2-1732253750847 heartbeating to localhost/127.0.0.1:33625 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:36:07,578 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:36:07,578 WARN  [BP-2132726750-172.17.0.2-1732253750847 heartbeating to localhost/127.0.0.1:33625 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2132726750-172.17.0.2-1732253750847 (Datanode Uuid ef78f60e-06a3-423f-8d31-3d9ef5a1ab0f) service to localhost/127.0.0.1:33625
2024-11-22T05:36:07,578 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:36:07,578 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/data/data1/current/BP-2132726750-172.17.0.2-1732253750847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:36:07,579 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/data/data2/current/BP-2132726750-172.17.0.2-1732253750847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:36:07,579 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:36:07,588 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:36:07,591 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:36:07,592 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:36:07,592 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:36:07,592 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:36:07,592 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@282cfbc9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:36:07,593 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4165df3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:36:07,668 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:07,668 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:07,694 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@75ed142f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/java.io.tmpdir/jetty-localhost-43083-hadoop-hdfs-3_4_1-tests_jar-_-any-11156112490621839161/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:36:07,694 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@78dcc37b{HTTP/1.1, (http/1.1)}{localhost:43083}
2024-11-22T05:36:07,694 INFO  [Time-limited test {}] server.Server(415): Started @174952ms
2024-11-22T05:36:07,696 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:36:08,077 WARN  [Thread-1347 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:36:08,079 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3586f715467809ff with lease ID 0x2fe2e139ffd6cbbc: from storage DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc node DatanodeRegistration(127.0.0.1:46517, datanodeUuid=e6df19ae-c2fa-4664-b025-3b09edab69c2, infoPort=35697, infoSecurePort=0, ipcPort=41045, storageInfo=lv=-57;cid=testClusterID;nsid=649795978;c=1732253750847), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:36:08,079 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3586f715467809ff with lease ID 0x2fe2e139ffd6cbbc: from storage DS-c3110d24-b5eb-4d87-b263-958a883db4db node DatanodeRegistration(127.0.0.1:46517, datanodeUuid=e6df19ae-c2fa-4664-b025-3b09edab69c2, infoPort=35697, infoSecurePort=0, ipcPort=41045, storageInfo=lv=-57;cid=testClusterID;nsid=649795978;c=1732253750847), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:36:08,230 WARN  [Thread-1367 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:36:08,232 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x89032f7662b4ad3a with lease ID 0x2fe2e139ffd6cbbd: from storage DS-ba03f590-e634-4a24-9cd0-1da6a86808c4 node DatanodeRegistration(127.0.0.1:43995, datanodeUuid=ef78f60e-06a3-423f-8d31-3d9ef5a1ab0f, infoPort=38067, infoSecurePort=0, ipcPort=36589, storageInfo=lv=-57;cid=testClusterID;nsid=649795978;c=1732253750847), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:36:08,232 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x89032f7662b4ad3a with lease ID 0x2fe2e139ffd6cbbd: from storage DS-ae8e6145-0240-4bcc-9edc-c9d02319350f node DatanodeRegistration(127.0.0.1:43995, datanodeUuid=ef78f60e-06a3-423f-8d31-3d9ef5a1ab0f, infoPort=38067, infoSecurePort=0, ipcPort=36589, storageInfo=lv=-57;cid=testClusterID;nsid=649795978;c=1732253750847), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:36:08,668 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:08,669 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:08,715 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-11-22T05:36:08,719 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-11-22T05:36:08,720 ERROR [FSHLog-0-hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4-prefix:82891a793a29,39785,1732253753525 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43725,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:08,721 WARN  [FSHLog-0-hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4-prefix:82891a793a29,39785,1732253753525 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43725,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:08,721 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 82891a793a29%2C39785%2C1732253753525:(num 1732253754448) roll requested
2024-11-22T05:36:08,721 INFO  [regionserver/82891a793a29:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C39785%2C1732253753525.1732253768721
2024-11-22T05:36:08,727 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253754448 newFile=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721
2024-11-22T05:36:08,727 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:08,727 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:08,727 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:08,727 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:08,727 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:08,727 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253754448 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721
2024-11-22T05:36:08,728 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43725,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:08,728 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43725,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:08,728 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253754448
2024-11-22T05:36:08,728 WARN  [IPC Server handler 4 on default port 33625 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253754448 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1015
2024-11-22T05:36:08,729 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253754448 after 1ms
2024-11-22T05:36:08,729 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35697:35697),(127.0.0.1/127.0.0.1:38067:38067)]
2024-11-22T05:36:08,729 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253754448 is not closed yet, will try archiving it next time
2024-11-22T05:36:09,669 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:09,669 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:10,079 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1015: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-22T05:36:10,670 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:10,670 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:10,733 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-11-22T05:36:11,671 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:11,671 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:12,671 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:12,671 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:12,729 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253754448 after 4001ms
2024-11-22T05:36:12,748 WARN  [ResponseProcessor for block BP-2132726750-172.17.0.2-1732253750847:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2132726750-172.17.0.2-1732253750847:blk_1073741837_1016
java.io.IOException: Bad response ERROR for BP-2132726750-172.17.0.2-1732253750847:blk_1073741837_1016 from datanode DatanodeInfoWithStorage[127.0.0.1:43995,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:12,748 WARN  [DataStreamer for file /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721 block BP-2132726750-172.17.0.2-1732253750847:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2132726750-172.17.0.2-1732253750847:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46517,DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc,DISK], DatanodeInfoWithStorage[127.0.0.1:43995,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43995,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]) is bad.
2024-11-22T05:36:12,749 WARN  [PacketResponder: BP-2132726750-172.17.0.2-1732253750847:blk_1073741837_1016, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:43995] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:12,749 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_70066984_22 at /127.0.0.1:43482 [Receiving block BP-2132726750-172.17.0.2-1732253750847:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:46517:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43482 dst: /127.0.0.1:46517
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:12,749 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_70066984_22 at /127.0.0.1:49126 [Receiving block BP-2132726750-172.17.0.2-1732253750847:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:43995:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49126 dst: /127.0.0.1:43995
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:12,794 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@75ed142f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:36:12,794 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@78dcc37b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:36:12,794 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:36:12,794 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4165df3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:36:12,794 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@282cfbc9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.log.dir/,STOPPED}
2024-11-22T05:36:12,796 WARN  [BP-2132726750-172.17.0.2-1732253750847 heartbeating to localhost/127.0.0.1:33625 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:36:12,796 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:36:12,796 WARN  [BP-2132726750-172.17.0.2-1732253750847 heartbeating to localhost/127.0.0.1:33625 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2132726750-172.17.0.2-1732253750847 (Datanode Uuid ef78f60e-06a3-423f-8d31-3d9ef5a1ab0f) service to localhost/127.0.0.1:33625
2024-11-22T05:36:12,796 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:36:12,796 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/data/data1/current/BP-2132726750-172.17.0.2-1732253750847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:36:12,797 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/data/data2/current/BP-2132726750-172.17.0.2-1732253750847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:36:12,797 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:36:12,806 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:36:12,810 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:36:12,812 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:36:12,812 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:36:12,812 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:36:12,813 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@73f17350{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:36:12,813 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22586a60{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:36:12,918 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@19016e01{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/java.io.tmpdir/jetty-localhost-41179-hadoop-hdfs-3_4_1-tests_jar-_-any-3022963021401727484/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:36:12,918 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@701cf8bd{HTTP/1.1, (http/1.1)}{localhost:41179}
2024-11-22T05:36:12,918 INFO  [Time-limited test {}] server.Server(415): Started @180176ms
2024-11-22T05:36:12,919 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:36:12,948 WARN  [ResponseProcessor for block BP-2132726750-172.17.0.2-1732253750847:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2132726750-172.17.0.2-1732253750847:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:12,949 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_70066984_22 at /127.0.0.1:50984 [Receiving block BP-2132726750-172.17.0.2-1732253750847:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:46517:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50984 dst: /127.0.0.1:46517
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:12,950 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2460467f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:36:12,950 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a3c48a5{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:36:12,950 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:36:12,951 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48dc7e0c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:36:12,951 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70770496{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.log.dir/,STOPPED}
2024-11-22T05:36:12,952 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:36:12,952 WARN  [BP-2132726750-172.17.0.2-1732253750847 heartbeating to localhost/127.0.0.1:33625 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:36:12,952 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:36:12,952 WARN  [BP-2132726750-172.17.0.2-1732253750847 heartbeating to localhost/127.0.0.1:33625 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2132726750-172.17.0.2-1732253750847 (Datanode Uuid e6df19ae-c2fa-4664-b025-3b09edab69c2) service to localhost/127.0.0.1:33625
2024-11-22T05:36:12,953 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/data/data3/current/BP-2132726750-172.17.0.2-1732253750847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:36:12,953 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/data/data4/current/BP-2132726750-172.17.0.2-1732253750847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:36:12,953 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:36:12,975 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:36:12,979 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:36:12,984 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:36:12,984 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:36:12,984 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:36:12,988 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@379a3c90{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:36:12,988 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@636dd77c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:36:13,096 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6c39138a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/java.io.tmpdir/jetty-localhost-45059-hadoop-hdfs-3_4_1-tests_jar-_-any-1515326203919491608/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:36:13,097 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3eee2fd1{HTTP/1.1, (http/1.1)}{localhost:45059}
2024-11-22T05:36:13,097 INFO  [Time-limited test {}] server.Server(415): Started @180354ms
2024-11-22T05:36:13,098 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:36:13,403 WARN  [Thread-1421 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:36:13,405 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xee0b3e8d93c1b495 with lease ID 0x2fe2e139ffd6cbbe: from storage DS-ba03f590-e634-4a24-9cd0-1da6a86808c4 node DatanodeRegistration(127.0.0.1:33869, datanodeUuid=ef78f60e-06a3-423f-8d31-3d9ef5a1ab0f, infoPort=34517, infoSecurePort=0, ipcPort=43223, storageInfo=lv=-57;cid=testClusterID;nsid=649795978;c=1732253750847), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:36:13,406 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xee0b3e8d93c1b495 with lease ID 0x2fe2e139ffd6cbbe: from storage DS-ae8e6145-0240-4bcc-9edc-c9d02319350f node DatanodeRegistration(127.0.0.1:33869, datanodeUuid=ef78f60e-06a3-423f-8d31-3d9ef5a1ab0f, infoPort=34517, infoSecurePort=0, ipcPort=43223, storageInfo=lv=-57;cid=testClusterID;nsid=649795978;c=1732253750847), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:36:13,635 WARN  [Thread-1441 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:36:13,637 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x17f4a2a251bdc24d with lease ID 0x2fe2e139ffd6cbbf: from storage DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc node DatanodeRegistration(127.0.0.1:36017, datanodeUuid=e6df19ae-c2fa-4664-b025-3b09edab69c2, infoPort=35157, infoSecurePort=0, ipcPort=40243, storageInfo=lv=-57;cid=testClusterID;nsid=649795978;c=1732253750847), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-22T05:36:13,637 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x17f4a2a251bdc24d with lease ID 0x2fe2e139ffd6cbbf: from storage DS-c3110d24-b5eb-4d87-b263-958a883db4db node DatanodeRegistration(127.0.0.1:36017, datanodeUuid=e6df19ae-c2fa-4664-b025-3b09edab69c2, infoPort=35157, infoSecurePort=0, ipcPort=40243, storageInfo=lv=-57;cid=testClusterID;nsid=649795978;c=1732253750847), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:36:13,672 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:13,672 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:14,129 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-11-22T05:36:14,132 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-11-22T05:36:14,134 ERROR [FSHLog-0-hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4-prefix:82891a793a29,39785,1732253753525 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46517,DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:14,134 WARN  [FSHLog-0-hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4-prefix:82891a793a29,39785,1732253753525 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46517,DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:14,134 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 82891a793a29%2C39785%2C1732253753525:(num 1732253768721) roll requested
2024-11-22T05:36:14,135 INFO  [regionserver/82891a793a29:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C39785%2C1732253753525.1732253774135
2024-11-22T05:36:14,149 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721 newFile=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253774135
2024-11-22T05:36:14,149 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:14,149 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:14,149 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:14,149 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:14,149 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:14,149 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253774135
2024-11-22T05:36:14,150 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46517,DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:14,150 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46517,DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:14,150 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721
2024-11-22T05:36:14,150 WARN  [IPC Server handler 0 on default port 33625 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-11-22T05:36:14,150 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35157:35157),(127.0.0.1/127.0.0.1:34517:34517)]
2024-11-22T05:36:14,150 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721 is not closed yet, will try archiving it next time
2024-11-22T05:36:14,151 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721 after 0ms
2024-11-22T05:36:14,673 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:14,673 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:15,674 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:15,674 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:16,152 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C39785%2C1732253753525.1732253776152
2024-11-22T05:36:16,159 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253774135 newFile=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152
2024-11-22T05:36:16,159 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:16,160 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:16,160 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:16,160 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:16,160 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:16,160 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253774135 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152
2024-11-22T05:36:16,161 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35157:35157),(127.0.0.1/127.0.0.1:34517:34517)]
2024-11-22T05:36:16,161 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721 is not closed yet, will try archiving it next time
2024-11-22T05:36:16,162 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253774135 is not closed yet, will try archiving it next time
2024-11-22T05:36:16,162 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253754448
2024-11-22T05:36:16,162 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253754448
2024-11-22T05:36:16,162 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741838_1019 (size=1264)
2024-11-22T05:36:16,163 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253754448 after 1ms
2024-11-22T05:36:16,163 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36017 is added to blk_1073741838_1019 (size=1264)
2024-11-22T05:36:16,163 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253754448
2024-11-22T05:36:16,163 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721 is not closed yet, will try archiving it next time
2024-11-22T05:36:16,176 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1732253755432/Put/vlen=218/seqid=0]
2024-11-22T05:36:16,176 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1732253765183/Put/vlen=1045/seqid=0]
2024-11-22T05:36:16,176 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253754448
2024-11-22T05:36:16,176 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721
2024-11-22T05:36:16,176 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721
2024-11-22T05:36:16,177 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721 after 1ms
2024-11-22T05:36:16,177 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721
2024-11-22T05:36:16,181 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1732253768720/Put/vlen=1045/seqid=0]
2024-11-22T05:36:16,181 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1732253770735/Put/vlen=1045/seqid=0]
2024-11-22T05:36:16,181 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721
2024-11-22T05:36:16,181 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253774135
2024-11-22T05:36:16,181 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253774135
2024-11-22T05:36:16,182 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253774135 after 1ms
2024-11-22T05:36:16,182 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253774135
2024-11-22T05:36:16,185 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1732253774134/Put/vlen=1045/seqid=0]
2024-11-22T05:36:16,185 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152
2024-11-22T05:36:16,185 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152
2024-11-22T05:36:16,186 WARN  [IPC Server handler 1 on default port 33625 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-11-22T05:36:16,186 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152 after 1ms
2024-11-22T05:36:16,406 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-22T05:36:16,638 WARN  [ResponseProcessor for block BP-2132726750-172.17.0.2-1732253750847:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2132726750-172.17.0.2-1732253750847:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:16,638 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_660166755_22 at /127.0.0.1:49776 [Receiving block BP-2132726750-172.17.0.2-1732253750847:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:36017:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49776 dst: /127.0.0.1:36017
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36017 remote=/127.0.0.1:49776]. Total timeout mills is 60000, 59521 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:16,638 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_660166755_22 at /127.0.0.1:56200 [Receiving block BP-2132726750-172.17.0.2-1732253750847:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:33869:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56200 dst: /127.0.0.1:33869
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:16,638 WARN  [DataStreamer for file /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152 block BP-2132726750-172.17.0.2-1732253750847:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2132726750-172.17.0.2-1732253750847:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36017,DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc,DISK], DatanodeInfoWithStorage[127.0.0.1:33869,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36017,DS-0f0c8124-2a66-4f32-ab12-e96a30df4cdc,DISK]) is bad.
2024-11-22T05:36:16,639 WARN  [DataStreamer for file /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152 block BP-2132726750-172.17.0.2-1732253750847:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2132726750-172.17.0.2-1732253750847:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:16,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36017 is added to blk_1073741839_1022 (size=85)
2024-11-22T05:36:16,645 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741839_1022 (size=85)
2024-11-22T05:36:16,675 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:16,675 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:17,676 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:17,676 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:18,152 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253768721 after 4002ms
2024-11-22T05:36:18,677 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:18,677 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:19,678 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:19,678 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:20,187 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152 after 4002ms
2024-11-22T05:36:20,187 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152
2024-11-22T05:36:20,192 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152
2024-11-22T05:36:20,192 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-11-22T05:36:20,193 ERROR [FSHLog-0-hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4-prefix:82891a793a29,39785,1732253753525.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43725,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:20,193 WARN  [FSHLog-0-hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4-prefix:82891a793a29,39785,1732253753525.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43725,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:20,193 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 82891a793a29%2C39785%2C1732253753525.meta:.meta(num 1732253754832) roll requested
2024-11-22T05:36:20,193 INFO  [regionserver/82891a793a29:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C39785%2C1732253753525.meta.1732253780193.meta
2024-11-22T05:36:20,199 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:20,199 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:20,199 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:20,199 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:20,200 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:20,200 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.meta.1732253754832.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.meta.1732253780193.meta
2024-11-22T05:36:20,200 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43725,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:20,200 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43725,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:20,200 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.meta.1732253754832.meta
2024-11-22T05:36:20,201 WARN  [IPC Server handler 0 on default port 33625 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.meta.1732253754832.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1014
2024-11-22T05:36:20,201 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.meta.1732253754832.meta after 1ms
2024-11-22T05:36:20,207 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34517:34517),(127.0.0.1/127.0.0.1:35157:35157)]
2024-11-22T05:36:20,207 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.meta.1732253754832.meta is not closed yet, will try archiving it next time
2024-11-22T05:36:20,223 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/.tmp/info/9c0573e4b10e42e8950add17ed4d474c is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7./info:regioninfo/1732253755437/Put/seqid=0
2024-11-22T05:36:20,232 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741841_1025 (size=7125)
2024-11-22T05:36:20,232 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36017 is added to blk_1073741841_1025 (size=7125)
2024-11-22T05:36:20,233 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/.tmp/info/9c0573e4b10e42e8950add17ed4d474c
2024-11-22T05:36:20,256 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/.tmp/ns/6f1f51a7563d4abf864c33690aec9891 is 43, key is default/ns:d/1732253754942/Put/seqid=0
2024-11-22T05:36:20,261 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36017 is added to blk_1073741842_1026 (size=5153)
2024-11-22T05:36:20,262 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741842_1026 (size=5153)
2024-11-22T05:36:20,262 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/.tmp/ns/6f1f51a7563d4abf864c33690aec9891
2024-11-22T05:36:20,282 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/.tmp/table/edf952c553c74f72bc72a49fb0810d78 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1732253755448/Put/seqid=0
2024-11-22T05:36:20,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741843_1027 (size=5438)
2024-11-22T05:36:20,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36017 is added to blk_1073741843_1027 (size=5438)
2024-11-22T05:36:20,288 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/.tmp/table/edf952c553c74f72bc72a49fb0810d78
2024-11-22T05:36:20,294 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/.tmp/info/9c0573e4b10e42e8950add17ed4d474c as hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/info/9c0573e4b10e42e8950add17ed4d474c
2024-11-22T05:36:20,300 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/info/9c0573e4b10e42e8950add17ed4d474c, entries=10, sequenceid=11, filesize=7.0 K
2024-11-22T05:36:20,301 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/.tmp/ns/6f1f51a7563d4abf864c33690aec9891 as hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/ns/6f1f51a7563d4abf864c33690aec9891
2024-11-22T05:36:20,307 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/ns/6f1f51a7563d4abf864c33690aec9891, entries=2, sequenceid=11, filesize=5.0 K
2024-11-22T05:36:20,308 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/.tmp/table/edf952c553c74f72bc72a49fb0810d78 as hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/table/edf952c553c74f72bc72a49fb0810d78
2024-11-22T05:36:20,314 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/table/edf952c553c74f72bc72a49fb0810d78, entries=2, sequenceid=11, filesize=5.3 K
2024-11-22T05:36:20,315 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 123ms, sequenceid=11, compaction requested=false
2024-11-22T05:36:20,315 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-22T05:36:20,316 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 8e4107d9b30a70518c33762ff0d1bbc7 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-22T05:36:20,316 ERROR [FSHLog-0-hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4-prefix:82891a793a29,39785,1732253753525 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2132726750-172.17.0.2-1732253750847:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:20,316 WARN  [FSHLog-0-hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4-prefix:82891a793a29,39785,1732253753525 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2132726750-172.17.0.2-1732253750847:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:20,317 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 82891a793a29%2C39785%2C1732253753525:(num 1732253776152) roll requested
2024-11-22T05:36:20,317 INFO  [regionserver/82891a793a29:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C39785%2C1732253753525.1732253780317
2024-11-22T05:36:20,322 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152 newFile=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253780317
2024-11-22T05:36:20,322 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:20,322 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:20,322 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:20,322 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:20,322 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:20,322 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253780317
2024-11-22T05:36:20,322 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2132726750-172.17.0.2-1732253750847:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:20,323 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2132726750-172.17.0.2-1732253750847:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:20,323 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152
2024-11-22T05:36:20,323 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35157:35157),(127.0.0.1/127.0.0.1:34517:34517)]
2024-11-22T05:36:20,323 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152 after 0ms
2024-11-22T05:36:20,323 DEBUG [regionserver/82891a793a29:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152 is not closed yet, will try archiving it next time
2024-11-22T05:36:20,324 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.1732253776152 to hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/oldWALs/82891a793a29%2C39785%2C1732253753525.1732253776152
2024-11-22T05:36:20,344 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/default/TestLogRolling-testLogRollOnPipelineRestart/8e4107d9b30a70518c33762ff0d1bbc7/.tmp/info/d0d671eb8776486499af90ce2c106031 is 1080, key is row1002/info:/1732253765183/Put/seqid=0
2024-11-22T05:36:20,349 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741845_1029 (size=9270)
2024-11-22T05:36:20,349 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36017 is added to blk_1073741845_1029 (size=9270)
2024-11-22T05:36:20,350 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/default/TestLogRolling-testLogRollOnPipelineRestart/8e4107d9b30a70518c33762ff0d1bbc7/.tmp/info/d0d671eb8776486499af90ce2c106031
2024-11-22T05:36:20,356 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/default/TestLogRolling-testLogRollOnPipelineRestart/8e4107d9b30a70518c33762ff0d1bbc7/.tmp/info/d0d671eb8776486499af90ce2c106031 as hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/default/TestLogRolling-testLogRollOnPipelineRestart/8e4107d9b30a70518c33762ff0d1bbc7/info/d0d671eb8776486499af90ce2c106031
2024-11-22T05:36:20,364 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/default/TestLogRolling-testLogRollOnPipelineRestart/8e4107d9b30a70518c33762ff0d1bbc7/info/d0d671eb8776486499af90ce2c106031, entries=4, sequenceid=8, filesize=9.1 K
2024-11-22T05:36:20,366 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 8e4107d9b30a70518c33762ff0d1bbc7 in 49ms, sequenceid=8, compaction requested=false
2024-11-22T05:36:20,366 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8e4107d9b30a70518c33762ff0d1bbc7:

2024-11-22T05:36:20,371 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-22T05:36:20,371 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T05:36:20,371 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:36:20,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:36:20,371 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-22T05:36:20,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:36:20,371 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-22T05:36:20,371 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=207666871, stopped=false
2024-11-22T05:36:20,371 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=82891a793a29,37779,1732253753357
2024-11-22T05:36:20,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T05:36:20,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T05:36:20,437 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T05:36:20,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:36:20,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:36:20,437 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T05:36:20,437 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:36:20,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:36:20,438 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '82891a793a29,39785,1732253753525' *****
2024-11-22T05:36:20,438 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-22T05:36:20,438 INFO  [RS:0;82891a793a29:39785 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-22T05:36:20,438 INFO  [RS:0;82891a793a29:39785 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-22T05:36:20,438 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-22T05:36:20,438 INFO  [RS:0;82891a793a29:39785 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-22T05:36:20,438 INFO  [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer(3091): Received CLOSE for 8e4107d9b30a70518c33762ff0d1bbc7
2024-11-22T05:36:20,438 INFO  [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer(959): stopping server 82891a793a29,39785,1732253753525
2024-11-22T05:36:20,438 INFO  [RS:0;82891a793a29:39785 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T05:36:20,439 INFO  [RS:0;82891a793a29:39785 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;82891a793a29:39785.
2024-11-22T05:36:20,439 DEBUG [RS:0;82891a793a29:39785 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:36:20,439 DEBUG [RS:0;82891a793a29:39785 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:36:20,439 INFO  [RS:0;82891a793a29:39785 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-22T05:36:20,439 INFO  [RS:0;82891a793a29:39785 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-22T05:36:20,439 INFO  [RS:0;82891a793a29:39785 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-22T05:36:20,439 INFO  [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-22T05:36:20,439 INFO  [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-22T05:36:20,439 DEBUG [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 8e4107d9b30a70518c33762ff0d1bbc7=TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.}
2024-11-22T05:36:20,439 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8e4107d9b30a70518c33762ff0d1bbc7, disabling compactions & flushes
2024-11-22T05:36:20,439 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T05:36:20,439 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T05:36:20,439 DEBUG [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 8e4107d9b30a70518c33762ff0d1bbc7
2024-11-22T05:36:20,439 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.
2024-11-22T05:36:20,439 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T05:36:20,439 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.
2024-11-22T05:36:20,440 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T05:36:20,440 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7. after waiting 0 ms
2024-11-22T05:36:20,440 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T05:36:20,440 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.
2024-11-22T05:36:20,440 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:36:20,441 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:36:20,449 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/default/TestLogRolling-testLogRollOnPipelineRestart/8e4107d9b30a70518c33762ff0d1bbc7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-11-22T05:36:20,450 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.
2024-11-22T05:36:20,450 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8e4107d9b30a70518c33762ff0d1bbc7:
Waiting for close lock at 1732253780439Running coprocessor pre-close hooks at 1732253780439Disabling compacts and flushes for region at 1732253780439Disabling writes for close at 1732253780440 (+1 ms)Writing region close event to WAL at 1732253780444 (+4 ms)Running coprocessor post-close hooks at 1732253780450 (+6 ms)Closed at 1732253780450
2024-11-22T05:36:20,450 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732253755071.8e4107d9b30a70518c33762ff0d1bbc7.
2024-11-22T05:36:20,451 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-22T05:36:20,452 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T05:36:20,452 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T05:36:20,452 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732253780439Running coprocessor pre-close hooks at 1732253780439Disabling compacts and flushes for region at 1732253780439Disabling writes for close at 1732253780440 (+1 ms)Writing region close event to WAL at 1732253780449 (+9 ms)Running coprocessor post-close hooks at 1732253780452 (+3 ms)Closed at 1732253780452
2024-11-22T05:36:20,452 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-22T05:36:20,640 INFO  [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer(976): stopping server 82891a793a29,39785,1732253753525; all regions closed.
2024-11-22T05:36:20,640 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:20,641 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:20,641 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:20,641 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:20,641 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:20,643 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36017 is added to blk_1073741840_1023 (size=825)
2024-11-22T05:36:20,643 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741840_1023 (size=825)
2024-11-22T05:36:20,678 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:20,679 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:21,312 INFO  [regionserver/82891a793a29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-22T05:36:21,312 INFO  [regionserver/82891a793a29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-22T05:36:21,679 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:21,679 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:22,312 INFO  [regionserver/82891a793a29:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T05:36:22,637 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1014: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-22T05:36:22,680 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:22,680 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:23,340 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-22T05:36:23,680 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:23,680 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:24,202 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.meta.1732253754832.meta after 4002ms
2024-11-22T05:36:24,202 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/WALs/82891a793a29,39785,1732253753525/82891a793a29%2C39785%2C1732253753525.meta.1732253754832.meta to hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/oldWALs/82891a793a29%2C39785%2C1732253753525.meta.1732253754832.meta
2024-11-22T05:36:24,205 DEBUG [RS:0;82891a793a29:39785 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/oldWALs
2024-11-22T05:36:24,205 INFO  [RS:0;82891a793a29:39785 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 82891a793a29%2C39785%2C1732253753525.meta:.meta(num 1732253780193)
2024-11-22T05:36:24,205 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:24,205 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:24,206 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:24,206 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:24,206 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:24,207 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741844_1028 (size=1162)
2024-11-22T05:36:24,207 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36017 is added to blk_1073741844_1028 (size=1162)
2024-11-22T05:36:24,212 DEBUG [RS:0;82891a793a29:39785 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/oldWALs
2024-11-22T05:36:24,212 INFO  [RS:0;82891a793a29:39785 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 82891a793a29%2C39785%2C1732253753525:(num 1732253780317)
2024-11-22T05:36:24,212 DEBUG [RS:0;82891a793a29:39785 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:36:24,212 INFO  [RS:0;82891a793a29:39785 {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T05:36:24,212 INFO  [RS:0;82891a793a29:39785 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T05:36:24,213 INFO  [RS:0;82891a793a29:39785 {}] hbase.ChoreService(370): Chore service for: regionserver/82891a793a29:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-22T05:36:24,213 INFO  [RS:0;82891a793a29:39785 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T05:36:24,213 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T05:36:24,213 INFO  [RS:0;82891a793a29:39785 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39785
2024-11-22T05:36:24,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T05:36:24,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/82891a793a29,39785,1732253753525
2024-11-22T05:36:24,258 INFO  [RS:0;82891a793a29:39785 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T05:36:24,258 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [82891a793a29,39785,1732253753525]
2024-11-22T05:36:24,278 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/82891a793a29,39785,1732253753525 already deleted, retry=false
2024-11-22T05:36:24,279 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 82891a793a29,39785,1732253753525 expired; onlineServers=0
2024-11-22T05:36:24,279 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '82891a793a29,37779,1732253753357' *****
2024-11-22T05:36:24,279 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-22T05:36:24,279 INFO  [M:0;82891a793a29:37779 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T05:36:24,279 INFO  [M:0;82891a793a29:37779 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T05:36:24,279 DEBUG [M:0;82891a793a29:37779 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-22T05:36:24,279 DEBUG [M:0;82891a793a29:37779 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-22T05:36:24,279 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-22T05:36:24,279 DEBUG [master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253754201 {}] cleaner.HFileCleaner(306): Exit Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253754201,5,FailOnTimeoutGroup]
2024-11-22T05:36:24,279 DEBUG [master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253754201 {}] cleaner.HFileCleaner(306): Exit Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253754201,5,FailOnTimeoutGroup]
2024-11-22T05:36:24,280 INFO  [M:0;82891a793a29:37779 {}] hbase.ChoreService(370): Chore service for: master/82891a793a29:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-22T05:36:24,280 INFO  [M:0;82891a793a29:37779 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T05:36:24,280 DEBUG [M:0;82891a793a29:37779 {}] master.HMaster(1795): Stopping service threads
2024-11-22T05:36:24,280 INFO  [M:0;82891a793a29:37779 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-22T05:36:24,280 INFO  [M:0;82891a793a29:37779 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T05:36:24,280 INFO  [M:0;82891a793a29:37779 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-22T05:36:24,280 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-22T05:36:24,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-22T05:36:24,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:36:24,289 DEBUG [M:0;82891a793a29:37779 {}] zookeeper.ZKUtil(347): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-22T05:36:24,289 WARN  [M:0;82891a793a29:37779 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-22T05:36:24,290 INFO  [M:0;82891a793a29:37779 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/.lastflushedseqids
2024-11-22T05:36:24,301 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36017 is added to blk_1073741846_1030 (size=111)
2024-11-22T05:36:24,301 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741846_1030 (size=111)
2024-11-22T05:36:24,301 INFO  [M:0;82891a793a29:37779 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-22T05:36:24,301 INFO  [M:0;82891a793a29:37779 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-22T05:36:24,301 DEBUG [M:0;82891a793a29:37779 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T05:36:24,301 INFO  [M:0;82891a793a29:37779 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:36:24,302 DEBUG [M:0;82891a793a29:37779 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:36:24,302 DEBUG [M:0;82891a793a29:37779 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T05:36:24,302 DEBUG [M:0;82891a793a29:37779 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:36:24,302 INFO  [M:0;82891a793a29:37779 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB
2024-11-22T05:36:24,302 ERROR [FSHLog-0-hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData-prefix:82891a793a29,37779,1732253753357 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43725,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:24,302 WARN  [FSHLog-0-hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData-prefix:82891a793a29,37779,1732253753357 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43725,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:24,302 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 82891a793a29%2C37779%2C1732253753357:(num 1732253753673) roll requested
2024-11-22T05:36:24,303 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C37779%2C1732253753357.1732253784303
2024-11-22T05:36:24,315 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:24,315 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:24,315 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:24,315 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:24,315 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:24,315 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/WALs/82891a793a29,37779,1732253753357/82891a793a29%2C37779%2C1732253753357.1732253753673 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/WALs/82891a793a29,37779,1732253753357/82891a793a29%2C37779%2C1732253753357.1732253784303
2024-11-22T05:36:24,315 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43725,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:24,316 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43725,DS-ba03f590-e634-4a24-9cd0-1da6a86808c4,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T05:36:24,316 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/WALs/82891a793a29,37779,1732253753357/82891a793a29%2C37779%2C1732253753357.1732253753673
2024-11-22T05:36:24,316 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34517:34517),(127.0.0.1/127.0.0.1:35157:35157)]
2024-11-22T05:36:24,316 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/WALs/82891a793a29,37779,1732253753357/82891a793a29%2C37779%2C1732253753357.1732253753673 is not closed yet, will try archiving it next time
2024-11-22T05:36:24,316 WARN  [IPC Server handler 1 on default port 33625 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/WALs/82891a793a29,37779,1732253753357/82891a793a29%2C37779%2C1732253753357.1732253753673 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1013
2024-11-22T05:36:24,316 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/WALs/82891a793a29,37779,1732253753357/82891a793a29%2C37779%2C1732253753357.1732253753673 after 0ms
2024-11-22T05:36:24,331 DEBUG [M:0;82891a793a29:37779 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ddbdcf0b6c7b4e4287404e4849966230 is 82, key is hbase:meta,,1/info:regioninfo/1732253754862/Put/seqid=0
2024-11-22T05:36:24,336 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741848_1033 (size=5672)
2024-11-22T05:36:24,336 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36017 is added to blk_1073741848_1033 (size=5672)
2024-11-22T05:36:24,337 INFO  [M:0;82891a793a29:37779 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ddbdcf0b6c7b4e4287404e4849966230
2024-11-22T05:36:24,357 DEBUG [M:0;82891a793a29:37779 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6c54a56f7446483495b3af1c3b2151b2 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732253755454/Put/seqid=0
2024-11-22T05:36:24,362 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36017 is added to blk_1073741849_1034 (size=6118)
2024-11-22T05:36:24,362 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741849_1034 (size=6118)
2024-11-22T05:36:24,362 INFO  [M:0;82891a793a29:37779 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6c54a56f7446483495b3af1c3b2151b2
2024-11-22T05:36:24,368 INFO  [RS:0;82891a793a29:39785 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T05:36:24,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:36:24,368 INFO  [RS:0;82891a793a29:39785 {}] regionserver.HRegionServer(1031): Exiting; stopping=82891a793a29,39785,1732253753525; zookeeper connection closed.
2024-11-22T05:36:24,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39785-0x101610a9ba90001, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:36:24,369 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7e49d7be {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7e49d7be
2024-11-22T05:36:24,369 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-22T05:36:24,382 DEBUG [M:0;82891a793a29:37779 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/52d0b54818ab44468331cf5007759a0a is 69, key is 82891a793a29,39785,1732253753525/rs:state/1732253754290/Put/seqid=0
2024-11-22T05:36:24,387 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741850_1035 (size=5156)
2024-11-22T05:36:24,387 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36017 is added to blk_1073741850_1035 (size=5156)
2024-11-22T05:36:24,388 INFO  [M:0;82891a793a29:37779 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/52d0b54818ab44468331cf5007759a0a
2024-11-22T05:36:24,408 DEBUG [M:0;82891a793a29:37779 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4aae4810a85a42f796726d4a58c54a54 is 52, key is load_balancer_on/state:d/1732253755062/Put/seqid=0
2024-11-22T05:36:24,413 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36017 is added to blk_1073741851_1036 (size=5056)
2024-11-22T05:36:24,413 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741851_1036 (size=5056)
2024-11-22T05:36:24,413 INFO  [M:0;82891a793a29:37779 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4aae4810a85a42f796726d4a58c54a54
2024-11-22T05:36:24,426 DEBUG [M:0;82891a793a29:37779 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ddbdcf0b6c7b4e4287404e4849966230 as hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ddbdcf0b6c7b4e4287404e4849966230
2024-11-22T05:36:24,436 INFO  [M:0;82891a793a29:37779 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ddbdcf0b6c7b4e4287404e4849966230, entries=8, sequenceid=56, filesize=5.5 K
2024-11-22T05:36:24,437 DEBUG [M:0;82891a793a29:37779 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6c54a56f7446483495b3af1c3b2151b2 as hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6c54a56f7446483495b3af1c3b2151b2
2024-11-22T05:36:24,444 INFO  [M:0;82891a793a29:37779 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6c54a56f7446483495b3af1c3b2151b2, entries=6, sequenceid=56, filesize=6.0 K
2024-11-22T05:36:24,445 DEBUG [M:0;82891a793a29:37779 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/52d0b54818ab44468331cf5007759a0a as hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/52d0b54818ab44468331cf5007759a0a
2024-11-22T05:36:24,451 INFO  [M:0;82891a793a29:37779 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/52d0b54818ab44468331cf5007759a0a, entries=1, sequenceid=56, filesize=5.0 K
2024-11-22T05:36:24,452 DEBUG [M:0;82891a793a29:37779 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4aae4810a85a42f796726d4a58c54a54 as hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/4aae4810a85a42f796726d4a58c54a54
2024-11-22T05:36:24,458 INFO  [M:0;82891a793a29:37779 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/4aae4810a85a42f796726d4a58c54a54, entries=1, sequenceid=56, filesize=4.9 K
2024-11-22T05:36:24,459 INFO  [M:0;82891a793a29:37779 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 157ms, sequenceid=56, compaction requested=false
2024-11-22T05:36:24,461 INFO  [M:0;82891a793a29:37779 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:36:24,461 DEBUG [M:0;82891a793a29:37779 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732253784301Disabling compacts and flushes for region at 1732253784301Disabling writes for close at 1732253784302 (+1 ms)Obtaining lock to block concurrent updates at 1732253784302Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732253784302Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1732253784302Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732253784316 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732253784316Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732253784331 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732253784331Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732253784342 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732253784357 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732253784357Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732253784367 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732253784382 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732253784382Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732253784392 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732253784408 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732253784408Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f8c4235: reopening flushed file at 1732253784425 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ee5bfdf: reopening flushed file at 1732253784436 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f504c61: reopening flushed file at 1732253784444 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d60d530: reopening flushed file at 1732253784451 (+7 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 157ms, sequenceid=56, compaction requested=false at 1732253784459 (+8 ms)Writing region close event to WAL at 1732253784461 (+2 ms)Closed at 1732253784461
2024-11-22T05:36:24,461 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:24,461 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:24,461 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:24,462 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:24,462 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:36:24,474 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36017 is added to blk_1073741847_1031 (size=757)
2024-11-22T05:36:24,475 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741847_1031 (size=757)
2024-11-22T05:36:24,681 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:24,681 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:25,450 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,450 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,467 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,468 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,468 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,468 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,468 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,468 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,473 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,473 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,473 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,475 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,480 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,480 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,638 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1013: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-22T05:36:25,682 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:25,682 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:25,982 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T05:36:25,984 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,984 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,985 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:25,985 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:26,007 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:26,008 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:26,008 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:26,008 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:26,008 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:26,009 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:26,014 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:26,014 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:26,014 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:26,017 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:26,683 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:26,683 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:27,684 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:27,684 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:28,317 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/WALs/82891a793a29,37779,1732253753357/82891a793a29%2C37779%2C1732253753357.1732253753673 after 4001ms
2024-11-22T05:36:28,318 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/WALs/82891a793a29,37779,1732253753357/82891a793a29%2C37779%2C1732253753357.1732253753673 to hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/oldWALs/82891a793a29%2C37779%2C1732253753357.1732253753673
2024-11-22T05:36:28,322 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/MasterData/oldWALs/82891a793a29%2C37779%2C1732253753357.1732253753673 to hdfs://localhost:33625/user/jenkins/test-data/ace57616-fd0a-168c-54e8-9ce9f6588df4/oldWALs/82891a793a29%2C37779%2C1732253753357.1732253753673$masterlocalwal$
2024-11-22T05:36:28,322 INFO  [M:0;82891a793a29:37779 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-22T05:36:28,322 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T05:36:28,322 INFO  [M:0;82891a793a29:37779 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37779
2024-11-22T05:36:28,322 INFO  [M:0;82891a793a29:37779 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T05:36:28,431 INFO  [M:0;82891a793a29:37779 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T05:36:28,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:36:28,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37779-0x101610a9ba90000, quorum=127.0.0.1:51421, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:36:28,434 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6c39138a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:36:28,434 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3eee2fd1{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:36:28,434 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:36:28,434 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@636dd77c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:36:28,434 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@379a3c90{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.log.dir/,STOPPED}
2024-11-22T05:36:28,436 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:36:28,436 WARN  [BP-2132726750-172.17.0.2-1732253750847 heartbeating to localhost/127.0.0.1:33625 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:36:28,436 WARN  [BP-2132726750-172.17.0.2-1732253750847 heartbeating to localhost/127.0.0.1:33625 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2132726750-172.17.0.2-1732253750847 (Datanode Uuid e6df19ae-c2fa-4664-b025-3b09edab69c2) service to localhost/127.0.0.1:33625
2024-11-22T05:36:28,436 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:36:28,436 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/data/data3/current/BP-2132726750-172.17.0.2-1732253750847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:36:28,437 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/data/data4/current/BP-2132726750-172.17.0.2-1732253750847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:36:28,437 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:36:28,439 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@19016e01{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:36:28,439 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@701cf8bd{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:36:28,440 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:36:28,440 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22586a60{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:36:28,440 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@73f17350{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.log.dir/,STOPPED}
2024-11-22T05:36:28,441 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:36:28,441 WARN  [BP-2132726750-172.17.0.2-1732253750847 heartbeating to localhost/127.0.0.1:33625 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:36:28,441 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:36:28,441 WARN  [BP-2132726750-172.17.0.2-1732253750847 heartbeating to localhost/127.0.0.1:33625 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2132726750-172.17.0.2-1732253750847 (Datanode Uuid ef78f60e-06a3-423f-8d31-3d9ef5a1ab0f) service to localhost/127.0.0.1:33625
2024-11-22T05:36:28,442 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/data/data1/current/BP-2132726750-172.17.0.2-1732253750847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:36:28,442 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/cluster_227557c0-f776-e847-7533-dd8c4f5cf9c6/data/data2/current/BP-2132726750-172.17.0.2-1732253750847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:36:28,442 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:36:28,448 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@42369482{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T05:36:28,449 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@159bbbac{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:36:28,449 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:36:28,449 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78e57521{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:36:28,449 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5493c38f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.log.dir/,STOPPED}
2024-11-22T05:36:28,456 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-22T05:36:28,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-22T05:36:28,485 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=180 (was 154)
Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:33625
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33625
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33625 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33625 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33625 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:33625
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Command processor
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1411)
	app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33625
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33625
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=221 (was 246), ProcessCount=11 (was 11), AvailableMemoryMB=7169 (was 7366)
2024-11-22T05:36:28,493 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=180, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=221, ProcessCount=11, AvailableMemoryMB=7169
2024-11-22T05:36:28,493 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-22T05:36:28,493 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.log.dir so I do NOT create it in target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded
2024-11-22T05:36:28,493 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a65dd470-dbbd-89f3-8c8a-c79f38e494f6/hadoop.tmp.dir so I do NOT create it in target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded
2024-11-22T05:36:28,494 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/cluster_88355450-062b-87e6-5738-3243a3ec2fb3, deleteOnExit=true
2024-11-22T05:36:28,494 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-22T05:36:28,494 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/test.cache.data in system properties and HBase conf
2024-11-22T05:36:28,494 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/hadoop.tmp.dir in system properties and HBase conf
2024-11-22T05:36:28,494 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/hadoop.log.dir in system properties and HBase conf
2024-11-22T05:36:28,494 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-22T05:36:28,494 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-22T05:36:28,494 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-22T05:36:28,494 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-22T05:36:28,495 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-22T05:36:28,495 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-22T05:36:28,495 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-22T05:36:28,495 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T05:36:28,495 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-22T05:36:28,495 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-22T05:36:28,495 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T05:36:28,495 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T05:36:28,495 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-22T05:36:28,495 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/nfs.dump.dir in system properties and HBase conf
2024-11-22T05:36:28,495 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/java.io.tmpdir in system properties and HBase conf
2024-11-22T05:36:28,495 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T05:36:28,495 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-22T05:36:28,496 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-22T05:36:28,508 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T05:36:28,685 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:28,685 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:28,918 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:36:28,922 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:36:28,923 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:36:28,923 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:36:28,923 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T05:36:28,924 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:36:28,924 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4e960720{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:36:28,924 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@9cd716d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:36:29,026 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6da95783{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/java.io.tmpdir/jetty-localhost-34589-hadoop-hdfs-3_4_1-tests_jar-_-any-13853213846327272997/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T05:36:29,026 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@293e66d4{HTTP/1.1, (http/1.1)}{localhost:34589}
2024-11-22T05:36:29,026 INFO  [Time-limited test {}] server.Server(415): Started @196284ms
2024-11-22T05:36:29,040 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T05:36:29,298 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:36:29,301 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:36:29,302 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:36:29,302 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:36:29,302 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:36:29,303 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b3ac0d9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:36:29,303 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@424c779{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:36:29,406 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3be31a0b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/java.io.tmpdir/jetty-localhost-38885-hadoop-hdfs-3_4_1-tests_jar-_-any-16511123566312720439/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:36:29,406 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2cfed7b4{HTTP/1.1, (http/1.1)}{localhost:38885}
2024-11-22T05:36:29,406 INFO  [Time-limited test {}] server.Server(415): Started @196664ms
2024-11-22T05:36:29,407 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:36:29,437 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:36:29,440 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:36:29,441 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:36:29,441 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:36:29,441 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T05:36:29,441 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47ccf9d3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:36:29,442 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49c1230f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:36:29,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T05:36:29,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-22T05:36:29,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-22T05:36:29,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-22T05:36:29,549 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1beefc80{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/java.io.tmpdir/jetty-localhost-35615-hadoop-hdfs-3_4_1-tests_jar-_-any-14522545116406617266/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:36:29,549 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10c2896a{HTTP/1.1, (http/1.1)}{localhost:35615}
2024-11-22T05:36:29,549 INFO  [Time-limited test {}] server.Server(415): Started @196807ms
2024-11-22T05:36:29,550 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:36:29,685 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:29,685 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:30,686 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:30,686 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:30,822 WARN  [Thread-1661 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/cluster_88355450-062b-87e6-5738-3243a3ec2fb3/data/data1/current/BP-163586823-172.17.0.2-1732253788519/current, will proceed with Du for space computation calculation, 
2024-11-22T05:36:30,822 WARN  [Thread-1662 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/cluster_88355450-062b-87e6-5738-3243a3ec2fb3/data/data2/current/BP-163586823-172.17.0.2-1732253788519/current, will proceed with Du for space computation calculation, 
2024-11-22T05:36:30,838 WARN  [Thread-1625 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:36:30,841 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdeb9013ebecd16ea with lease ID 0x734024f75d050f79: Processing first storage report for DS-bd43ab24-c813-4197-9a85-af94bcae4a3f from datanode DatanodeRegistration(127.0.0.1:41219, datanodeUuid=3757d4af-3767-4c1d-8089-57b6e355f415, infoPort=42837, infoSecurePort=0, ipcPort=41419, storageInfo=lv=-57;cid=testClusterID;nsid=887170405;c=1732253788519)
2024-11-22T05:36:30,841 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdeb9013ebecd16ea with lease ID 0x734024f75d050f79: from storage DS-bd43ab24-c813-4197-9a85-af94bcae4a3f node DatanodeRegistration(127.0.0.1:41219, datanodeUuid=3757d4af-3767-4c1d-8089-57b6e355f415, infoPort=42837, infoSecurePort=0, ipcPort=41419, storageInfo=lv=-57;cid=testClusterID;nsid=887170405;c=1732253788519), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:36:30,841 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdeb9013ebecd16ea with lease ID 0x734024f75d050f79: Processing first storage report for DS-cc1912c3-866b-42a4-afae-5ddd58d4ed76 from datanode DatanodeRegistration(127.0.0.1:41219, datanodeUuid=3757d4af-3767-4c1d-8089-57b6e355f415, infoPort=42837, infoSecurePort=0, ipcPort=41419, storageInfo=lv=-57;cid=testClusterID;nsid=887170405;c=1732253788519)
2024-11-22T05:36:30,841 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdeb9013ebecd16ea with lease ID 0x734024f75d050f79: from storage DS-cc1912c3-866b-42a4-afae-5ddd58d4ed76 node DatanodeRegistration(127.0.0.1:41219, datanodeUuid=3757d4af-3767-4c1d-8089-57b6e355f415, infoPort=42837, infoSecurePort=0, ipcPort=41419, storageInfo=lv=-57;cid=testClusterID;nsid=887170405;c=1732253788519), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:36:30,952 WARN  [Thread-1672 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/cluster_88355450-062b-87e6-5738-3243a3ec2fb3/data/data3/current/BP-163586823-172.17.0.2-1732253788519/current, will proceed with Du for space computation calculation, 
2024-11-22T05:36:30,953 WARN  [Thread-1673 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/cluster_88355450-062b-87e6-5738-3243a3ec2fb3/data/data4/current/BP-163586823-172.17.0.2-1732253788519/current, will proceed with Du for space computation calculation, 
2024-11-22T05:36:30,974 WARN  [Thread-1648 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:36:30,976 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2db741adbc0f8040 with lease ID 0x734024f75d050f7a: Processing first storage report for DS-990f6aa5-4338-48e5-a8ba-810be6f13de4 from datanode DatanodeRegistration(127.0.0.1:43975, datanodeUuid=153ca1ff-2a5b-4914-95db-5b124d576104, infoPort=34981, infoSecurePort=0, ipcPort=33539, storageInfo=lv=-57;cid=testClusterID;nsid=887170405;c=1732253788519)
2024-11-22T05:36:30,976 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2db741adbc0f8040 with lease ID 0x734024f75d050f7a: from storage DS-990f6aa5-4338-48e5-a8ba-810be6f13de4 node DatanodeRegistration(127.0.0.1:43975, datanodeUuid=153ca1ff-2a5b-4914-95db-5b124d576104, infoPort=34981, infoSecurePort=0, ipcPort=33539, storageInfo=lv=-57;cid=testClusterID;nsid=887170405;c=1732253788519), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:36:30,976 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2db741adbc0f8040 with lease ID 0x734024f75d050f7a: Processing first storage report for DS-dfec4736-8f30-4e8e-99c4-035c96dfc4ca from datanode DatanodeRegistration(127.0.0.1:43975, datanodeUuid=153ca1ff-2a5b-4914-95db-5b124d576104, infoPort=34981, infoSecurePort=0, ipcPort=33539, storageInfo=lv=-57;cid=testClusterID;nsid=887170405;c=1732253788519)
2024-11-22T05:36:30,976 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2db741adbc0f8040 with lease ID 0x734024f75d050f7a: from storage DS-dfec4736-8f30-4e8e-99c4-035c96dfc4ca node DatanodeRegistration(127.0.0.1:43975, datanodeUuid=153ca1ff-2a5b-4914-95db-5b124d576104, infoPort=34981, infoSecurePort=0, ipcPort=33539, storageInfo=lv=-57;cid=testClusterID;nsid=887170405;c=1732253788519), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:36:30,982 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded
2024-11-22T05:36:30,986 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/cluster_88355450-062b-87e6-5738-3243a3ec2fb3/zookeeper_0, clientPort=62440, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/cluster_88355450-062b-87e6-5738-3243a3ec2fb3/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/cluster_88355450-062b-87e6-5738-3243a3ec2fb3/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-22T05:36:30,987 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62440
2024-11-22T05:36:30,987 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:36:30,989 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:36:30,996 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741825_1001 (size=7)
2024-11-22T05:36:30,997 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741825_1001 (size=7)
2024-11-22T05:36:30,998 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1 with version=8
2024-11-22T05:36:30,998 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/hbase-staging
2024-11-22T05:36:30,999 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/82891a793a29:0 server-side Connection retries=45
2024-11-22T05:36:30,999 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:36:31,000 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T05:36:31,000 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T05:36:31,000 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:36:31,000 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T05:36:31,000 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-22T05:36:31,000 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T05:36:31,000 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44241
2024-11-22T05:36:31,002 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44241 connecting to ZooKeeper ensemble=127.0.0.1:62440
2024-11-22T05:36:31,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:442410x0, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T05:36:31,053 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44241-0x101610b2eb50000 connected
2024-11-22T05:36:31,131 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:36:31,133 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:36:31,135 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:36:31,136 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1, hbase.cluster.distributed=false
2024-11-22T05:36:31,138 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T05:36:31,138 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44241
2024-11-22T05:36:31,138 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44241
2024-11-22T05:36:31,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44241
2024-11-22T05:36:31,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44241
2024-11-22T05:36:31,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44241
2024-11-22T05:36:31,157 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/82891a793a29:0 server-side Connection retries=45
2024-11-22T05:36:31,157 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:36:31,157 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T05:36:31,157 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T05:36:31,157 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:36:31,157 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T05:36:31,157 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-22T05:36:31,157 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T05:36:31,158 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42251
2024-11-22T05:36:31,159 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42251 connecting to ZooKeeper ensemble=127.0.0.1:62440
2024-11-22T05:36:31,160 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:36:31,162 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:36:31,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422510x0, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T05:36:31,173 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:422510x0, quorum=127.0.0.1:62440, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:36:31,173 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42251-0x101610b2eb50001 connected
2024-11-22T05:36:31,174 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-22T05:36:31,174 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-22T05:36:31,175 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-22T05:36:31,176 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T05:36:31,176 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42251
2024-11-22T05:36:31,176 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42251
2024-11-22T05:36:31,177 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42251
2024-11-22T05:36:31,177 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42251
2024-11-22T05:36:31,177 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42251
2024-11-22T05:36:31,189 DEBUG [M:0;82891a793a29:44241 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;82891a793a29:44241
2024-11-22T05:36:31,189 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/82891a793a29,44241,1732253790999
2024-11-22T05:36:31,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:36:31,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:36:31,194 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/82891a793a29,44241,1732253790999
2024-11-22T05:36:31,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-22T05:36:31,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:36:31,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:36:31,205 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-22T05:36:31,205 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/82891a793a29,44241,1732253790999 from backup master directory
2024-11-22T05:36:31,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:36:31,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/82891a793a29,44241,1732253790999
2024-11-22T05:36:31,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:36:31,215 WARN  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T05:36:31,215 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=82891a793a29,44241,1732253790999
2024-11-22T05:36:31,221 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/hbase.id] with ID: a7f662c6-0c22-46b3-bc98-37ea1ee37210
2024-11-22T05:36:31,221 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/.tmp/hbase.id
2024-11-22T05:36:31,230 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741826_1002 (size=42)
2024-11-22T05:36:31,230 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741826_1002 (size=42)
2024-11-22T05:36:31,231 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/.tmp/hbase.id]:[hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/hbase.id]
2024-11-22T05:36:31,242 INFO  [master/82891a793a29:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:36:31,242 INFO  [master/82891a793a29:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-22T05:36:31,243 INFO  [master/82891a793a29:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-22T05:36:31,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:36:31,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:36:31,268 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741827_1003 (size=196)
2024-11-22T05:36:31,268 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741827_1003 (size=196)
2024-11-22T05:36:31,269 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T05:36:31,270 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-22T05:36:31,270 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:36:31,277 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741828_1004 (size=1189)
2024-11-22T05:36:31,277 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741828_1004 (size=1189)
2024-11-22T05:36:31,278 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store
2024-11-22T05:36:31,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741829_1005 (size=34)
2024-11-22T05:36:31,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741829_1005 (size=34)
2024-11-22T05:36:31,288 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:36:31,288 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T05:36:31,288 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:36:31,288 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:36:31,288 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T05:36:31,288 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:36:31,288 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:36:31,288 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732253791288Disabling compacts and flushes for region at 1732253791288Disabling writes for close at 1732253791288Writing region close event to WAL at 1732253791288Closed at 1732253791288
2024-11-22T05:36:31,289 WARN  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/.initializing
2024-11-22T05:36:31,289 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/WALs/82891a793a29,44241,1732253790999
2024-11-22T05:36:31,291 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C44241%2C1732253790999, suffix=, logDir=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/WALs/82891a793a29,44241,1732253790999, archiveDir=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/oldWALs, maxLogs=10
2024-11-22T05:36:31,292 INFO  [master/82891a793a29:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C44241%2C1732253790999.1732253791292
2024-11-22T05:36:31,296 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/WALs/82891a793a29,44241,1732253790999/82891a793a29%2C44241%2C1732253790999.1732253791292
2024-11-22T05:36:31,297 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34981:34981),(127.0.0.1/127.0.0.1:42837:42837)]
2024-11-22T05:36:31,298 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:36:31,298 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:36:31,298 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:36:31,298 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:36:31,300 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:36:31,301 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-22T05:36:31,301 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:36:31,301 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:36:31,301 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:36:31,302 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-22T05:36:31,302 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:36:31,303 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:36:31,303 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:36:31,304 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-22T05:36:31,304 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:36:31,304 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:36:31,304 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:36:31,306 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-22T05:36:31,306 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:36:31,306 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:36:31,307 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:36:31,307 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:36:31,308 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:36:31,309 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:36:31,309 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:36:31,310 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-22T05:36:31,311 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:36:31,313 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:36:31,314 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=772039, jitterRate=-0.018302276730537415}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-22T05:36:31,315 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732253791298Initializing all the Stores at 1732253791299 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253791299Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253791299Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253791299Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253791299Cleaning up temporary data from old regions at 1732253791309 (+10 ms)Region opened successfully at 1732253791315 (+6 ms)
2024-11-22T05:36:31,315 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-22T05:36:31,318 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f0fd16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82891a793a29/172.17.0.2:0
2024-11-22T05:36:31,319 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-22T05:36:31,319 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-22T05:36:31,320 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-22T05:36:31,320 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-22T05:36:31,320 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-22T05:36:31,321 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-22T05:36:31,321 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-22T05:36:31,323 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-22T05:36:31,324 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-22T05:36:31,331 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-22T05:36:31,331 INFO  [master/82891a793a29:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-22T05:36:31,332 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-22T05:36:31,341 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-22T05:36:31,342 INFO  [master/82891a793a29:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-22T05:36:31,343 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-22T05:36:31,352 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-22T05:36:31,353 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-22T05:36:31,362 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-22T05:36:31,365 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-22T05:36:31,373 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-22T05:36:31,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T05:36:31,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T05:36:31,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:36:31,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:36:31,384 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=82891a793a29,44241,1732253790999, sessionid=0x101610b2eb50000, setting cluster-up flag (Was=false)
2024-11-22T05:36:31,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:36:31,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:36:31,436 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-22T05:36:31,437 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=82891a793a29,44241,1732253790999
2024-11-22T05:36:31,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:36:31,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:36:31,489 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-22T05:36:31,490 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=82891a793a29,44241,1732253790999
2024-11-22T05:36:31,491 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-22T05:36:31,493 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-22T05:36:31,493 INFO  [master/82891a793a29:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-22T05:36:31,493 INFO  [master/82891a793a29:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-22T05:36:31,494 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 82891a793a29,44241,1732253790999
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-22T05:36:31,495 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:36:31,495 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:36:31,495 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:36:31,495 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:36:31,496 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/82891a793a29:0, corePoolSize=10, maxPoolSize=10
2024-11-22T05:36:31,496 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:36:31,496 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/82891a793a29:0, corePoolSize=2, maxPoolSize=2
2024-11-22T05:36:31,496 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:36:31,497 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732253821497
2024-11-22T05:36:31,497 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-22T05:36:31,497 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-22T05:36:31,497 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-22T05:36:31,497 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-22T05:36:31,497 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-22T05:36:31,497 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-22T05:36:31,497 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:31,498 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-22T05:36:31,498 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-22T05:36:31,498 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-22T05:36:31,498 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:36:31,498 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-22T05:36:31,498 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-22T05:36:31,498 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-22T05:36:31,498 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253791498,5,FailOnTimeoutGroup]
2024-11-22T05:36:31,499 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253791498,5,FailOnTimeoutGroup]
2024-11-22T05:36:31,499 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:31,499 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-22T05:36:31,499 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:31,499 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:31,499 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:36:31,499 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-22T05:36:31,507 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741831_1007 (size=1321)
2024-11-22T05:36:31,507 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741831_1007 (size=1321)
2024-11-22T05:36:31,509 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-22T05:36:31,509 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1
2024-11-22T05:36:31,518 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741832_1008 (size=32)
2024-11-22T05:36:31,518 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741832_1008 (size=32)
2024-11-22T05:36:31,519 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:36:31,520 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T05:36:31,521 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T05:36:31,522 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:36:31,522 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:36:31,522 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T05:36:31,523 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T05:36:31,524 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:36:31,524 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:36:31,524 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T05:36:31,525 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T05:36:31,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:36:31,526 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:36:31,526 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T05:36:31,527 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T05:36:31,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:36:31,528 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:36:31,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T05:36:31,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740
2024-11-22T05:36:31,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740
2024-11-22T05:36:31,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T05:36:31,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T05:36:31,530 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T05:36:31,532 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T05:36:31,534 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:36:31,534 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=849878, jitterRate=0.0806766003370285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T05:36:31,535 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732253791519Initializing all the Stores at 1732253791520 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253791520Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253791520Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253791520Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253791520Cleaning up temporary data from old regions at 1732253791530 (+10 ms)Region opened successfully at 1732253791535 (+5 ms)
2024-11-22T05:36:31,535 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T05:36:31,535 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T05:36:31,535 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T05:36:31,535 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T05:36:31,535 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T05:36:31,536 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T05:36:31,536 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732253791535Disabling compacts and flushes for region at 1732253791535Disabling writes for close at 1732253791535Writing region close event to WAL at 1732253791536 (+1 ms)Closed at 1732253791536
2024-11-22T05:36:31,537 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:36:31,537 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-22T05:36:31,537 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-22T05:36:31,539 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T05:36:31,540 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-22T05:36:31,580 INFO  [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer(746): ClusterId : a7f662c6-0c22-46b3-bc98-37ea1ee37210
2024-11-22T05:36:31,580 DEBUG [RS:0;82891a793a29:42251 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-22T05:36:31,594 DEBUG [RS:0;82891a793a29:42251 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-22T05:36:31,594 DEBUG [RS:0;82891a793a29:42251 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-22T05:36:31,605 DEBUG [RS:0;82891a793a29:42251 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-22T05:36:31,606 DEBUG [RS:0;82891a793a29:42251 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47feb791, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82891a793a29/172.17.0.2:0
2024-11-22T05:36:31,623 DEBUG [RS:0;82891a793a29:42251 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;82891a793a29:42251
2024-11-22T05:36:31,623 INFO  [RS:0;82891a793a29:42251 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-22T05:36:31,623 INFO  [RS:0;82891a793a29:42251 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-22T05:36:31,623 DEBUG [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-22T05:36:31,624 INFO  [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer(2659): reportForDuty to master=82891a793a29,44241,1732253790999 with port=42251, startcode=1732253791157
2024-11-22T05:36:31,624 DEBUG [RS:0;82891a793a29:42251 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-22T05:36:31,626 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43771, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-11-22T05:36:31,626 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44241 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 82891a793a29,42251,1732253791157
2024-11-22T05:36:31,627 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44241 {}] master.ServerManager(517): Registering regionserver=82891a793a29,42251,1732253791157
2024-11-22T05:36:31,628 DEBUG [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1
2024-11-22T05:36:31,628 DEBUG [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33769
2024-11-22T05:36:31,628 DEBUG [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-22T05:36:31,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T05:36:31,637 DEBUG [RS:0;82891a793a29:42251 {}] zookeeper.ZKUtil(111): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/82891a793a29,42251,1732253791157
2024-11-22T05:36:31,637 WARN  [RS:0;82891a793a29:42251 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T05:36:31,637 INFO  [RS:0;82891a793a29:42251 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:36:31,637 DEBUG [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/WALs/82891a793a29,42251,1732253791157
2024-11-22T05:36:31,637 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [82891a793a29,42251,1732253791157]
2024-11-22T05:36:31,641 INFO  [RS:0;82891a793a29:42251 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-22T05:36:31,643 INFO  [RS:0;82891a793a29:42251 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-22T05:36:31,643 INFO  [RS:0;82891a793a29:42251 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T05:36:31,643 INFO  [RS:0;82891a793a29:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:31,644 INFO  [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-22T05:36:31,645 INFO  [RS:0;82891a793a29:42251 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-22T05:36:31,645 INFO  [RS:0;82891a793a29:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:31,645 DEBUG [RS:0;82891a793a29:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:36:31,645 DEBUG [RS:0;82891a793a29:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:36:31,645 DEBUG [RS:0;82891a793a29:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:36:31,645 DEBUG [RS:0;82891a793a29:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:36:31,645 DEBUG [RS:0;82891a793a29:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:36:31,645 DEBUG [RS:0;82891a793a29:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/82891a793a29:0, corePoolSize=2, maxPoolSize=2
2024-11-22T05:36:31,645 DEBUG [RS:0;82891a793a29:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:36:31,645 DEBUG [RS:0;82891a793a29:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:36:31,645 DEBUG [RS:0;82891a793a29:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:36:31,645 DEBUG [RS:0;82891a793a29:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:36:31,645 DEBUG [RS:0;82891a793a29:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:36:31,645 DEBUG [RS:0;82891a793a29:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:36:31,645 DEBUG [RS:0;82891a793a29:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/82891a793a29:0, corePoolSize=3, maxPoolSize=3
2024-11-22T05:36:31,646 DEBUG [RS:0;82891a793a29:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0, corePoolSize=3, maxPoolSize=3
2024-11-22T05:36:31,646 INFO  [RS:0;82891a793a29:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:31,646 INFO  [RS:0;82891a793a29:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:31,646 INFO  [RS:0;82891a793a29:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:31,646 INFO  [RS:0;82891a793a29:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:31,646 INFO  [RS:0;82891a793a29:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:31,646 INFO  [RS:0;82891a793a29:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,42251,1732253791157-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T05:36:31,664 INFO  [RS:0;82891a793a29:42251 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-22T05:36:31,664 INFO  [RS:0;82891a793a29:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,42251,1732253791157-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:31,664 INFO  [RS:0;82891a793a29:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:31,664 INFO  [RS:0;82891a793a29:42251 {}] regionserver.Replication(171): 82891a793a29,42251,1732253791157 started
2024-11-22T05:36:31,677 INFO  [RS:0;82891a793a29:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:31,677 INFO  [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer(1482): Serving as 82891a793a29,42251,1732253791157, RpcServer on 82891a793a29/172.17.0.2:42251, sessionid=0x101610b2eb50001
2024-11-22T05:36:31,677 DEBUG [RS:0;82891a793a29:42251 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-22T05:36:31,677 DEBUG [RS:0;82891a793a29:42251 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 82891a793a29,42251,1732253791157
2024-11-22T05:36:31,677 DEBUG [RS:0;82891a793a29:42251 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82891a793a29,42251,1732253791157'
2024-11-22T05:36:31,677 DEBUG [RS:0;82891a793a29:42251 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-22T05:36:31,678 DEBUG [RS:0;82891a793a29:42251 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-22T05:36:31,678 DEBUG [RS:0;82891a793a29:42251 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-22T05:36:31,678 DEBUG [RS:0;82891a793a29:42251 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-22T05:36:31,678 DEBUG [RS:0;82891a793a29:42251 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 82891a793a29,42251,1732253791157
2024-11-22T05:36:31,678 DEBUG [RS:0;82891a793a29:42251 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82891a793a29,42251,1732253791157'
2024-11-22T05:36:31,678 DEBUG [RS:0;82891a793a29:42251 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-22T05:36:31,678 DEBUG [RS:0;82891a793a29:42251 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-22T05:36:31,679 DEBUG [RS:0;82891a793a29:42251 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-22T05:36:31,679 INFO  [RS:0;82891a793a29:42251 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-22T05:36:31,679 INFO  [RS:0;82891a793a29:42251 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-22T05:36:31,687 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:31,687 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:31,690 WARN  [82891a793a29:44241 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-22T05:36:31,781 INFO  [RS:0;82891a793a29:42251 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C42251%2C1732253791157, suffix=, logDir=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/WALs/82891a793a29,42251,1732253791157, archiveDir=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/oldWALs, maxLogs=32
2024-11-22T05:36:31,781 INFO  [RS:0;82891a793a29:42251 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42251%2C1732253791157.1732253791781
2024-11-22T05:36:31,786 INFO  [RS:0;82891a793a29:42251 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/WALs/82891a793a29,42251,1732253791157/82891a793a29%2C42251%2C1732253791157.1732253791781
2024-11-22T05:36:31,787 DEBUG [RS:0;82891a793a29:42251 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34981:34981),(127.0.0.1/127.0.0.1:42837:42837)]
2024-11-22T05:36:31,940 DEBUG [82891a793a29:44241 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-22T05:36:31,941 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=82891a793a29,42251,1732253791157
2024-11-22T05:36:31,942 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 82891a793a29,42251,1732253791157, state=OPENING
2024-11-22T05:36:31,952 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-22T05:36:31,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:36:31,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:36:31,963 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T05:36:31,963 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:36:31,963 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:36:31,963 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=82891a793a29,42251,1732253791157}]
2024-11-22T05:36:32,117 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-22T05:36:32,120 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46797, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-22T05:36:32,125 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-22T05:36:32,126 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:36:32,128 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C42251%2C1732253791157.meta, suffix=.meta, logDir=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/WALs/82891a793a29,42251,1732253791157, archiveDir=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/oldWALs, maxLogs=32
2024-11-22T05:36:32,129 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42251%2C1732253791157.meta.1732253792129.meta
2024-11-22T05:36:32,135 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/WALs/82891a793a29,42251,1732253791157/82891a793a29%2C42251%2C1732253791157.meta.1732253792129.meta
2024-11-22T05:36:32,136 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42837:42837),(127.0.0.1/127.0.0.1:34981:34981)]
2024-11-22T05:36:32,137 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:36:32,138 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-22T05:36:32,138 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-22T05:36:32,138 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-22T05:36:32,138 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-22T05:36:32,138 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:36:32,138 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-22T05:36:32,138 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-22T05:36:32,140 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T05:36:32,141 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T05:36:32,141 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:36:32,142 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:36:32,142 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T05:36:32,143 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T05:36:32,143 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:36:32,143 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:36:32,144 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T05:36:32,144 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T05:36:32,144 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:36:32,145 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:36:32,145 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T05:36:32,146 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T05:36:32,146 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:36:32,146 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:36:32,146 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T05:36:32,147 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740
2024-11-22T05:36:32,148 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740
2024-11-22T05:36:32,150 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T05:36:32,150 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T05:36:32,150 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T05:36:32,152 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T05:36:32,153 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=851335, jitterRate=0.08252920210361481}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T05:36:32,153 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-22T05:36:32,154 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732253792139Writing region info on filesystem at 1732253792139Initializing all the Stores at 1732253792140 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253792140Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253792140Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253792140Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253792140Cleaning up temporary data from old regions at 1732253792150 (+10 ms)Running coprocessor post-open hooks at 1732253792153 (+3 ms)Region opened successfully at 1732253792153
2024-11-22T05:36:32,155 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732253792117
2024-11-22T05:36:32,157 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-22T05:36:32,157 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-22T05:36:32,158 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=82891a793a29,42251,1732253791157
2024-11-22T05:36:32,160 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 82891a793a29,42251,1732253791157, state=OPEN
2024-11-22T05:36:32,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T05:36:32,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T05:36:32,201 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=82891a793a29,42251,1732253791157
2024-11-22T05:36:32,201 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:36:32,201 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:36:32,205 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-22T05:36:32,205 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=82891a793a29,42251,1732253791157 in 238 msec
2024-11-22T05:36:32,207 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-22T05:36:32,208 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 668 msec
2024-11-22T05:36:32,208 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:36:32,208 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-22T05:36:32,210 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T05:36:32,210 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82891a793a29,42251,1732253791157, seqNum=-1]
2024-11-22T05:36:32,210 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T05:36:32,212 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50417, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T05:36:32,218 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 724 msec
2024-11-22T05:36:32,218 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732253792218, completionTime=-1
2024-11-22T05:36:32,218 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-22T05:36:32,218 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-22T05:36:32,220 INFO  [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-22T05:36:32,220 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732253852220
2024-11-22T05:36:32,220 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732253912220
2024-11-22T05:36:32,220 INFO  [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-11-22T05:36:32,221 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,44241,1732253790999-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:32,221 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,44241,1732253790999-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:32,221 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,44241,1732253790999-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:32,221 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-82891a793a29:44241, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:32,221 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:32,221 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:32,223 DEBUG [master/82891a793a29:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-22T05:36:32,225 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.010sec
2024-11-22T05:36:32,225 INFO  [master/82891a793a29:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-22T05:36:32,225 INFO  [master/82891a793a29:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-22T05:36:32,225 INFO  [master/82891a793a29:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-22T05:36:32,225 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-22T05:36:32,225 INFO  [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-22T05:36:32,225 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,44241,1732253790999-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T05:36:32,226 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,44241,1732253790999-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-22T05:36:32,228 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-22T05:36:32,228 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-22T05:36:32,228 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,44241,1732253790999-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:36:32,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35c2cd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:36:32,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82891a793a29,44241,-1 for getting cluster id
2024-11-22T05:36:32,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-22T05:36:32,282 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7f662c6-0c22-46b3-bc98-37ea1ee37210'
2024-11-22T05:36:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-22T05:36:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7f662c6-0c22-46b3-bc98-37ea1ee37210"

2024-11-22T05:36:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b0c1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:36:32,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82891a793a29,44241,-1]
2024-11-22T05:36:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-22T05:36:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:36:32,284 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-22T05:36:32,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf94d5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:36:32,286 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T05:36:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82891a793a29,42251,1732253791157, seqNum=-1]
2024-11-22T05:36:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T05:36:32,288 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T05:36:32,290 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=82891a793a29,44241,1732253790999
2024-11-22T05:36:32,291 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:36:32,294 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-22T05:36:32,294 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-22T05:36:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 82891a793a29,44241,1732253790999
2024-11-22T05:36:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@542f684d
2024-11-22T05:36:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-22T05:36:32,297 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-22T05:36:32,297 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-22T05:36:32,297 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-22T05:36:32,298 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T05:36:32,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T05:36:32,300 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-22T05:36:32,301 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:36:32,301 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-11-22T05:36:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T05:36:32,302 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-22T05:36:32,309 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741835_1011 (size=405)
2024-11-22T05:36:32,309 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741835_1011 (size=405)
2024-11-22T05:36:32,317 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5a226cac55ef1679dd1da10570a74645, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1
2024-11-22T05:36:32,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741836_1012 (size=88)
2024-11-22T05:36:32,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741836_1012 (size=88)
2024-11-22T05:36:32,324 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:36:32,324 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 5a226cac55ef1679dd1da10570a74645, disabling compactions & flushes
2024-11-22T05:36:32,324 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:36:32,324 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:36:32,324 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645. after waiting 0 ms
2024-11-22T05:36:32,324 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:36:32,324 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:36:32,324 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5a226cac55ef1679dd1da10570a74645:
Waiting for close lock at 1732253792324Disabling compacts and flushes for region at 1732253792324Disabling writes for close at 1732253792324Writing region close event to WAL at 1732253792324Closed at 1732253792324
2024-11-22T05:36:32,325 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-22T05:36:32,326 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1732253792325"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732253792325"}]},"ts":"1732253792325"}
2024-11-22T05:36:32,328 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-22T05:36:32,329 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-22T05:36:32,330 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732253792329"}]},"ts":"1732253792329"}
2024-11-22T05:36:32,331 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-11-22T05:36:32,332 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=5a226cac55ef1679dd1da10570a74645, ASSIGN}]
2024-11-22T05:36:32,333 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=5a226cac55ef1679dd1da10570a74645, ASSIGN
2024-11-22T05:36:32,334 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=5a226cac55ef1679dd1da10570a74645, ASSIGN; state=OFFLINE, location=82891a793a29,42251,1732253791157; forceNewPlan=false, retain=false
2024-11-22T05:36:32,485 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5a226cac55ef1679dd1da10570a74645, regionState=OPENING, regionLocation=82891a793a29,42251,1732253791157
2024-11-22T05:36:32,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=5a226cac55ef1679dd1da10570a74645, ASSIGN because future has completed
2024-11-22T05:36:32,488 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a226cac55ef1679dd1da10570a74645, server=82891a793a29,42251,1732253791157}]
2024-11-22T05:36:32,647 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:36:32,648 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5a226cac55ef1679dd1da10570a74645, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:36:32,648 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 5a226cac55ef1679dd1da10570a74645
2024-11-22T05:36:32,648 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:36:32,649 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5a226cac55ef1679dd1da10570a74645
2024-11-22T05:36:32,649 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5a226cac55ef1679dd1da10570a74645
2024-11-22T05:36:32,651 INFO  [StoreOpener-5a226cac55ef1679dd1da10570a74645-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5a226cac55ef1679dd1da10570a74645 
2024-11-22T05:36:32,653 INFO  [StoreOpener-5a226cac55ef1679dd1da10570a74645-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a226cac55ef1679dd1da10570a74645 columnFamilyName info
2024-11-22T05:36:32,653 DEBUG [StoreOpener-5a226cac55ef1679dd1da10570a74645-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:36:32,654 INFO  [StoreOpener-5a226cac55ef1679dd1da10570a74645-1 {}] regionserver.HStore(327): Store=5a226cac55ef1679dd1da10570a74645/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:36:32,654 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5a226cac55ef1679dd1da10570a74645
2024-11-22T05:36:32,654 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645
2024-11-22T05:36:32,655 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645
2024-11-22T05:36:32,655 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5a226cac55ef1679dd1da10570a74645
2024-11-22T05:36:32,655 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5a226cac55ef1679dd1da10570a74645
2024-11-22T05:36:32,657 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5a226cac55ef1679dd1da10570a74645
2024-11-22T05:36:32,659 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:36:32,659 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5a226cac55ef1679dd1da10570a74645; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=692823, jitterRate=-0.11903093755245209}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-22T05:36:32,659 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5a226cac55ef1679dd1da10570a74645
2024-11-22T05:36:32,660 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5a226cac55ef1679dd1da10570a74645:
Running coprocessor pre-open hook at 1732253792649Writing region info on filesystem at 1732253792649Initializing all the Stores at 1732253792650 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253792650Cleaning up temporary data from old regions at 1732253792655 (+5 ms)Running coprocessor post-open hooks at 1732253792659 (+4 ms)Region opened successfully at 1732253792660 (+1 ms)
2024-11-22T05:36:32,661 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645., pid=6, masterSystemTime=1732253792641
2024-11-22T05:36:32,664 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:36:32,664 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:36:32,665 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5a226cac55ef1679dd1da10570a74645, regionState=OPEN, openSeqNum=2, regionLocation=82891a793a29,42251,1732253791157
2024-11-22T05:36:32,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a226cac55ef1679dd1da10570a74645, server=82891a793a29,42251,1732253791157 because future has completed
2024-11-22T05:36:32,672 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-22T05:36:32,672 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5a226cac55ef1679dd1da10570a74645, server=82891a793a29,42251,1732253791157 in 181 msec
2024-11-22T05:36:32,674 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-22T05:36:32,674 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=5a226cac55ef1679dd1da10570a74645, ASSIGN in 340 msec
2024-11-22T05:36:32,675 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-22T05:36:32,676 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732253792675"}]},"ts":"1732253792675"}
2024-11-22T05:36:32,678 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-11-22T05:36:32,679 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-22T05:36:32,681 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 381 msec
2024-11-22T05:36:32,688 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:32,688 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:33,688 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:33,688 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:34,689 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:34,689 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:34,994 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T05:36:34,995 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:34,996 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:34,996 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:34,996 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:34,997 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:34,997 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:35,019 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:35,019 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:35,019 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:35,020 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:35,020 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:35,020 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:35,024 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:35,025 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:35,025 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:35,028 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:36:35,690 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:35,690 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:36,690 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:36,690 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:37,641 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-22T05:36:37,642 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-11-22T05:36:37,691 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:37,691 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:38,692 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:38,692 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:39,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-22T05:36:39,491 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-22T05:36:39,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T05:36:39,493 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-22T05:36:39,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-22T05:36:39,493 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-22T05:36:39,494 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T05:36:39,494 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-11-22T05:36:39,692 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:39,692 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:40,693 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:40,693 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:41,694 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:41,694 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T05:36:42,354 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-22T05:36:42,354 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-11-22T05:36:42,358 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T05:36:42,358 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:36:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645., hostname=82891a793a29,42251,1732253791157, seqNum=2]
2024-11-22T05:36:42,371 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T05:36:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T05:36:42,378 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-22T05:36:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-22T05:36:42,380 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-22T05:36:42,381 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-22T05:36:42,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-11-22T05:36:42,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:36:42,544 INFO  [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 5a226cac55ef1679dd1da10570a74645 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-22T05:36:42,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/d4cb7986a0574847aff88314cfcd38f0 is 1080, key is row0001/info:/1732253802364/Put/seqid=0
2024-11-22T05:36:42,569 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741837_1013 (size=6033)
2024-11-22T05:36:42,570 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741837_1013 (size=6033)
2024-11-22T05:36:42,570 INFO  [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/d4cb7986a0574847aff88314cfcd38f0
2024-11-22T05:36:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/d4cb7986a0574847aff88314cfcd38f0 as hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/d4cb7986a0574847aff88314cfcd38f0
2024-11-22T05:36:42,583 INFO  [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/d4cb7986a0574847aff88314cfcd38f0, entries=1, sequenceid=5, filesize=5.9 K
2024-11-22T05:36:42,584 INFO  [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 5a226cac55ef1679dd1da10570a74645 in 40ms, sequenceid=5, compaction requested=false
2024-11-22T05:36:42,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 5a226cac55ef1679dd1da10570a74645:

2024-11-22T05:36:42,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:36:42,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-22T05:36:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-22T05:36:42,591 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-22T05:36:42,591 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 208 msec
2024-11-22T05:36:42,593 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 219 msec
2024-11-22T05:36:42,694 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:42,694 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:43,695 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:43,695 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:44,696 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:44,696 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:45,697 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:45,697 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:46,698 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:46,698 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:47,698 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:47,698 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:48,699 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:48,699 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:49,700 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:49,700 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:50,700 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:50,701 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:51,702 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:51,702 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-22T05:36:52,473 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-22T05:36:52,477 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T05:36:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T05:36:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-22T05:36:52,480 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-22T05:36:52,481 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-22T05:36:52,481 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-22T05:36:52,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-11-22T05:36:52,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:36:52,636 INFO  [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 5a226cac55ef1679dd1da10570a74645 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-22T05:36:52,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/8f048133178f47a2849fe4f68dd963b1 is 1080, key is row0002/info:/1732253812474/Put/seqid=0
2024-11-22T05:36:52,648 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741838_1014 (size=6033)
2024-11-22T05:36:52,648 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741838_1014 (size=6033)
2024-11-22T05:36:52,649 INFO  [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/8f048133178f47a2849fe4f68dd963b1
2024-11-22T05:36:52,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/8f048133178f47a2849fe4f68dd963b1 as hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/8f048133178f47a2849fe4f68dd963b1
2024-11-22T05:36:52,666 INFO  [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/8f048133178f47a2849fe4f68dd963b1, entries=1, sequenceid=9, filesize=5.9 K
2024-11-22T05:36:52,667 INFO  [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 5a226cac55ef1679dd1da10570a74645 in 32ms, sequenceid=9, compaction requested=false
2024-11-22T05:36:52,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 5a226cac55ef1679dd1da10570a74645:

2024-11-22T05:36:52,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:36:52,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-22T05:36:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-11-22T05:36:52,673 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-22T05:36:52,673 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 189 msec
2024-11-22T05:36:52,677 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 197 msec
2024-11-22T05:36:52,702 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:52,702 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:52,703 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319 after 68059ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:52,703 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta after 68048ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T05:36:53,704 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:53,704 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:54,704 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:54,704 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:55,705 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:55,705 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:56,706 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:56,706 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:57,706 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:57,706 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:58,707 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:58,707 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:59,708 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:36:59,708 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:00,708 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:00,708 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:00,981 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-22T05:37:01,709 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:01,709 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:02,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-22T05:37:02,544 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-22T05:37:02,548 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42251%2C1732253791157.1732253822548
2024-11-22T05:37:02,555 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:02,556 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:02,556 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:02,556 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:02,556 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:02,556 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/WALs/82891a793a29,42251,1732253791157/82891a793a29%2C42251%2C1732253791157.1732253791781 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/WALs/82891a793a29,42251,1732253791157/82891a793a29%2C42251%2C1732253791157.1732253822548
2024-11-22T05:37:02,558 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741833_1009 (size=5546)
2024-11-22T05:37:02,558 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741833_1009 (size=5546)
2024-11-22T05:37:02,561 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42837:42837),(127.0.0.1/127.0.0.1:34981:34981)]
2024-11-22T05:37:02,562 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T05:37:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T05:37:02,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-22T05:37:02,564 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-22T05:37:02,565 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-22T05:37:02,565 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-22T05:37:02,710 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:02,710 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:02,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-11-22T05:37:02,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:37:02,719 INFO  [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 5a226cac55ef1679dd1da10570a74645 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-22T05:37:02,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/235defe5a9e64b3690b1332a671145ce is 1080, key is row0003/info:/1732253822546/Put/seqid=0
2024-11-22T05:37:02,731 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741840_1016 (size=6033)
2024-11-22T05:37:02,731 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741840_1016 (size=6033)
2024-11-22T05:37:02,734 INFO  [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/235defe5a9e64b3690b1332a671145ce
2024-11-22T05:37:02,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/235defe5a9e64b3690b1332a671145ce as hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/235defe5a9e64b3690b1332a671145ce
2024-11-22T05:37:02,749 INFO  [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/235defe5a9e64b3690b1332a671145ce, entries=1, sequenceid=13, filesize=5.9 K
2024-11-22T05:37:02,750 INFO  [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 5a226cac55ef1679dd1da10570a74645 in 31ms, sequenceid=13, compaction requested=true
2024-11-22T05:37:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 5a226cac55ef1679dd1da10570a74645:

2024-11-22T05:37:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:37:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-22T05:37:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-11-22T05:37:02,754 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-22T05:37:02,754 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 187 msec
2024-11-22T05:37:02,757 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 193 msec
2024-11-22T05:37:03,711 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:03,711 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:04,712 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:04,712 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:05,713 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:05,713 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:06,713 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:06,713 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:07,714 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:07,714 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:08,715 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:08,715 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:09,716 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:09,716 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:10,716 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:10,716 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:11,717 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:11,717 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:12,240 INFO  [master/82891a793a29:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-22T05:37:12,240 INFO  [master/82891a793a29:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-22T05:37:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-22T05:37:12,643 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-22T05:37:12,643 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T05:37:12,645 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T05:37:12,645 DEBUG [Time-limited test {}] regionserver.HStore(1541): 5a226cac55ef1679dd1da10570a74645/info is initiating minor compaction (all files)
2024-11-22T05:37:12,645 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T05:37:12,645 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:12,645 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 5a226cac55ef1679dd1da10570a74645/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:37:12,645 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/d4cb7986a0574847aff88314cfcd38f0, hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/8f048133178f47a2849fe4f68dd963b1, hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/235defe5a9e64b3690b1332a671145ce] into tmpdir=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp, totalSize=17.7 K
2024-11-22T05:37:12,646 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d4cb7986a0574847aff88314cfcd38f0, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732253802364
2024-11-22T05:37:12,646 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8f048133178f47a2849fe4f68dd963b1, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1732253812474
2024-11-22T05:37:12,647 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 235defe5a9e64b3690b1332a671145ce, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732253822546
2024-11-22T05:37:12,658 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 5a226cac55ef1679dd1da10570a74645#info#compaction#45 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T05:37:12,659 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/0f9bbb1ace4546e2a575addc7aec2dff is 1080, key is row0001/info:/1732253802364/Put/seqid=0
2024-11-22T05:37:12,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741841_1017 (size=8296)
2024-11-22T05:37:12,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741841_1017 (size=8296)
2024-11-22T05:37:12,670 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/0f9bbb1ace4546e2a575addc7aec2dff as hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/0f9bbb1ace4546e2a575addc7aec2dff
2024-11-22T05:37:12,677 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a226cac55ef1679dd1da10570a74645/info of 5a226cac55ef1679dd1da10570a74645 into 0f9bbb1ace4546e2a575addc7aec2dff(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T05:37:12,677 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 5a226cac55ef1679dd1da10570a74645:

2024-11-22T05:37:12,680 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42251%2C1732253791157.1732253832680
2024-11-22T05:37:12,685 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:12,685 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:12,686 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:12,686 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:12,686 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:12,686 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/WALs/82891a793a29,42251,1732253791157/82891a793a29%2C42251%2C1732253791157.1732253822548 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/WALs/82891a793a29,42251,1732253791157/82891a793a29%2C42251%2C1732253791157.1732253832680
2024-11-22T05:37:12,687 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34981:34981),(127.0.0.1/127.0.0.1:42837:42837)]
2024-11-22T05:37:12,687 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/WALs/82891a793a29,42251,1732253791157/82891a793a29%2C42251%2C1732253791157.1732253822548 is not closed yet, will try archiving it next time
2024-11-22T05:37:12,687 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/WALs/82891a793a29,42251,1732253791157/82891a793a29%2C42251%2C1732253791157.1732253791781 to hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/oldWALs/82891a793a29%2C42251%2C1732253791157.1732253791781
2024-11-22T05:37:12,688 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T05:37:12,688 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741839_1015 (size=2520)
2024-11-22T05:37:12,688 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741839_1015 (size=2520)
2024-11-22T05:37:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T05:37:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-22T05:37:12,690 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-22T05:37:12,691 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-22T05:37:12,692 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-22T05:37:12,718 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:12,718 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:12,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-11-22T05:37:12,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:37:12,846 INFO  [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 5a226cac55ef1679dd1da10570a74645 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-22T05:37:12,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/d0818f2ea1b9415fa36bf16011f174be is 1080, key is row0000/info:/1732253832678/Put/seqid=0
2024-11-22T05:37:12,857 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741843_1019 (size=6033)
2024-11-22T05:37:12,858 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741843_1019 (size=6033)
2024-11-22T05:37:12,858 INFO  [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/d0818f2ea1b9415fa36bf16011f174be
2024-11-22T05:37:12,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/d0818f2ea1b9415fa36bf16011f174be as hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/d0818f2ea1b9415fa36bf16011f174be
2024-11-22T05:37:12,872 INFO  [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/d0818f2ea1b9415fa36bf16011f174be, entries=1, sequenceid=18, filesize=5.9 K
2024-11-22T05:37:12,873 INFO  [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 5a226cac55ef1679dd1da10570a74645 in 27ms, sequenceid=18, compaction requested=false
2024-11-22T05:37:12,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 5a226cac55ef1679dd1da10570a74645:

2024-11-22T05:37:12,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:37:12,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-22T05:37:12,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-11-22T05:37:12,877 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-22T05:37:12,877 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 183 msec
2024-11-22T05:37:12,879 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 190 msec
2024-11-22T05:37:13,718 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:13,718 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:14,719 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:14,719 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:15,720 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:15,720 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:16,721 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:16,721 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:17,649 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5a226cac55ef1679dd1da10570a74645, had cached 0 bytes from a total of 14329
2024-11-22T05:37:17,721 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:17,721 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:18,722 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:18,722 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:19,723 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:19,723 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:20,724 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:20,724 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:21,726 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:21,726 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-22T05:37:22,703 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-22T05:37:22,706 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42251%2C1732253791157.1732253842706
2024-11-22T05:37:22,712 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:22,712 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:22,713 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:22,713 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:22,713 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:22,713 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/WALs/82891a793a29,42251,1732253791157/82891a793a29%2C42251%2C1732253791157.1732253832680 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/WALs/82891a793a29,42251,1732253791157/82891a793a29%2C42251%2C1732253791157.1732253842706
2024-11-22T05:37:22,714 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34981:34981),(127.0.0.1/127.0.0.1:42837:42837)]
2024-11-22T05:37:22,714 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/WALs/82891a793a29,42251,1732253791157/82891a793a29%2C42251%2C1732253791157.1732253832680 is not closed yet, will try archiving it next time
2024-11-22T05:37:22,714 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-22T05:37:22,714 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/WALs/82891a793a29,42251,1732253791157/82891a793a29%2C42251%2C1732253791157.1732253822548 to hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/oldWALs/82891a793a29%2C42251%2C1732253791157.1732253822548
2024-11-22T05:37:22,714 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T05:37:22,714 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:37:22,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:37:22,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:37:22,715 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-22T05:37:22,715 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1262485124, stopped=false
2024-11-22T05:37:22,715 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741842_1018 (size=2026)
2024-11-22T05:37:22,715 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=82891a793a29,44241,1732253790999
2024-11-22T05:37:22,715 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-22T05:37:22,716 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741842_1018 (size=2026)
2024-11-22T05:37:22,726 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:22,726 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:22,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T05:37:22,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T05:37:22,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:37:22,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:37:22,760 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T05:37:22,760 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T05:37:22,760 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:37:22,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:37:22,761 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:37:22,761 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '82891a793a29,42251,1732253791157' *****
2024-11-22T05:37:22,761 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-22T05:37:22,761 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:37:22,761 INFO  [RS:0;82891a793a29:42251 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-22T05:37:22,761 INFO  [RS:0;82891a793a29:42251 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-22T05:37:22,761 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-22T05:37:22,761 INFO  [RS:0;82891a793a29:42251 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-22T05:37:22,761 INFO  [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer(3091): Received CLOSE for 5a226cac55ef1679dd1da10570a74645
2024-11-22T05:37:22,761 INFO  [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer(959): stopping server 82891a793a29,42251,1732253791157
2024-11-22T05:37:22,761 INFO  [RS:0;82891a793a29:42251 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T05:37:22,761 INFO  [RS:0;82891a793a29:42251 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;82891a793a29:42251.
2024-11-22T05:37:22,761 DEBUG [RS:0;82891a793a29:42251 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:37:22,761 DEBUG [RS:0;82891a793a29:42251 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:37:22,761 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5a226cac55ef1679dd1da10570a74645, disabling compactions & flushes
2024-11-22T05:37:22,761 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:37:22,761 INFO  [RS:0;82891a793a29:42251 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-22T05:37:22,762 INFO  [RS:0;82891a793a29:42251 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-22T05:37:22,762 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:37:22,762 INFO  [RS:0;82891a793a29:42251 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-22T05:37:22,762 INFO  [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-22T05:37:22,762 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645. after waiting 0 ms
2024-11-22T05:37:22,762 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:37:22,762 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5a226cac55ef1679dd1da10570a74645 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-22T05:37:22,762 INFO  [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-22T05:37:22,762 DEBUG [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 5a226cac55ef1679dd1da10570a74645=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.}
2024-11-22T05:37:22,762 DEBUG [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5a226cac55ef1679dd1da10570a74645
2024-11-22T05:37:22,762 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T05:37:22,762 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T05:37:22,762 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T05:37:22,762 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T05:37:22,762 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T05:37:22,762 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-11-22T05:37:22,765 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/ffbe549712bb49ffb487497037b47a22 is 1080, key is row0001/info:/1732253842705/Put/seqid=0
2024-11-22T05:37:22,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741845_1021 (size=6033)
2024-11-22T05:37:22,773 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741845_1021 (size=6033)
2024-11-22T05:37:22,773 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/ffbe549712bb49ffb487497037b47a22
2024-11-22T05:37:22,779 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/.tmp/info/b1f3e3ccdea349d082a57005564143bf is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645./info:regioninfo/1732253792665/Put/seqid=0
2024-11-22T05:37:22,779 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/.tmp/info/ffbe549712bb49ffb487497037b47a22 as hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/ffbe549712bb49ffb487497037b47a22
2024-11-22T05:37:22,784 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741846_1022 (size=7308)
2024-11-22T05:37:22,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741846_1022 (size=7308)
2024-11-22T05:37:22,785 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/ffbe549712bb49ffb487497037b47a22, entries=1, sequenceid=22, filesize=5.9 K
2024-11-22T05:37:22,785 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/.tmp/info/b1f3e3ccdea349d082a57005564143bf
2024-11-22T05:37:22,786 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 5a226cac55ef1679dd1da10570a74645 in 24ms, sequenceid=22, compaction requested=true
2024-11-22T05:37:22,788 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/d4cb7986a0574847aff88314cfcd38f0, hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/8f048133178f47a2849fe4f68dd963b1, hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/235defe5a9e64b3690b1332a671145ce] to archive
2024-11-22T05:37:22,789 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-22T05:37:22,791 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/d4cb7986a0574847aff88314cfcd38f0 to hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/d4cb7986a0574847aff88314cfcd38f0
2024-11-22T05:37:22,792 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/8f048133178f47a2849fe4f68dd963b1 to hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/8f048133178f47a2849fe4f68dd963b1
2024-11-22T05:37:22,793 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/235defe5a9e64b3690b1332a671145ce to hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/info/235defe5a9e64b3690b1332a671145ce
2024-11-22T05:37:22,793 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=82891a793a29:44241 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-22T05:37:22,794 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [d4cb7986a0574847aff88314cfcd38f0=6033, 8f048133178f47a2849fe4f68dd963b1=6033, 235defe5a9e64b3690b1332a671145ce=6033]
2024-11-22T05:37:22,798 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/5a226cac55ef1679dd1da10570a74645/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-11-22T05:37:22,799 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:37:22,799 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5a226cac55ef1679dd1da10570a74645:
Waiting for close lock at 1732253842761Running coprocessor pre-close hooks at 1732253842761Disabling compacts and flushes for region at 1732253842761Disabling writes for close at 1732253842762 (+1 ms)Obtaining lock to block concurrent updates at 1732253842762Preparing flush snapshotting stores in 5a226cac55ef1679dd1da10570a74645 at 1732253842762Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1732253842762Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645. at 1732253842763 (+1 ms)Flushing 5a226cac55ef1679dd1da10570a74645/info: creating writer at 1732253842763Flushing 5a226cac55ef1679dd1da10570a74645/info: appending metadata at 1732253842765 (+2 ms)Flushing 5a226cac55ef1679dd1da10570a74645/info: closing flushed file at 1732253842765Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2479f0c5: reopening flushed file at 1732253842778 (+13 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 5a226cac55ef1679dd1da10570a74645 in 24ms, sequenceid=22, compaction requested=true at 1732253842786 (+8 ms)Writing region close event to WAL at 1732253842794 (+8 ms)Running coprocessor post-close hooks at 1732253842799 (+5 ms)Closed at 1732253842799
2024-11-22T05:37:22,799 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732253792297.5a226cac55ef1679dd1da10570a74645.
2024-11-22T05:37:22,807 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/.tmp/ns/3766a847f18c4c2baa277a4e7177d65f is 43, key is default/ns:d/1732253792212/Put/seqid=0
2024-11-22T05:37:22,811 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741847_1023 (size=5153)
2024-11-22T05:37:22,812 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741847_1023 (size=5153)
2024-11-22T05:37:22,812 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/.tmp/ns/3766a847f18c4c2baa277a4e7177d65f
2024-11-22T05:37:22,830 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/.tmp/table/ecab7aee17d84a8f9e3d5f4650678b6e is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1732253792675/Put/seqid=0
2024-11-22T05:37:22,834 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741848_1024 (size=5508)
2024-11-22T05:37:22,834 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741848_1024 (size=5508)
2024-11-22T05:37:22,835 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/.tmp/table/ecab7aee17d84a8f9e3d5f4650678b6e
2024-11-22T05:37:22,840 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/.tmp/info/b1f3e3ccdea349d082a57005564143bf as hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/info/b1f3e3ccdea349d082a57005564143bf
2024-11-22T05:37:22,846 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/info/b1f3e3ccdea349d082a57005564143bf, entries=10, sequenceid=11, filesize=7.1 K
2024-11-22T05:37:22,847 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/.tmp/ns/3766a847f18c4c2baa277a4e7177d65f as hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/ns/3766a847f18c4c2baa277a4e7177d65f
2024-11-22T05:37:22,852 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/ns/3766a847f18c4c2baa277a4e7177d65f, entries=2, sequenceid=11, filesize=5.0 K
2024-11-22T05:37:22,853 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/.tmp/table/ecab7aee17d84a8f9e3d5f4650678b6e as hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/table/ecab7aee17d84a8f9e3d5f4650678b6e
2024-11-22T05:37:22,857 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/table/ecab7aee17d84a8f9e3d5f4650678b6e, entries=2, sequenceid=11, filesize=5.4 K
2024-11-22T05:37:22,858 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 96ms, sequenceid=11, compaction requested=false
2024-11-22T05:37:22,862 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-22T05:37:22,863 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T05:37:22,863 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T05:37:22,863 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732253842762Running coprocessor pre-close hooks at 1732253842762Disabling compacts and flushes for region at 1732253842762Disabling writes for close at 1732253842762Obtaining lock to block concurrent updates at 1732253842762Preparing flush snapshotting stores in 1588230740 at 1732253842762Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1732253842762Flushing stores of hbase:meta,,1.1588230740 at 1732253842763 (+1 ms)Flushing 1588230740/info: creating writer at 1732253842763Flushing 1588230740/info: appending metadata at 1732253842778 (+15 ms)Flushing 1588230740/info: closing flushed file at 1732253842778Flushing 1588230740/ns: creating writer at 1732253842791 (+13 ms)Flushing 1588230740/ns: appending metadata at 1732253842806 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732253842806Flushing 1588230740/table: creating writer at 1732253842817 (+11 ms)Flushing 1588230740/table: appending metadata at 1732253842830 (+13 ms)Flushing 1588230740/table: closing flushed file at 1732253842830Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b7d6725: reopening flushed file at 1732253842839 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aab90b7: reopening flushed file at 1732253842846 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@299ac66a: reopening flushed file at 1732253842852 (+6 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 96ms, sequenceid=11, compaction requested=false at 1732253842858 (+6 ms)Writing region close event to WAL at 1732253842859 (+1 ms)Running coprocessor post-close hooks at 1732253842862 (+3 ms)Closed at 1732253842863 (+1 ms)
2024-11-22T05:37:22,863 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-22T05:37:22,962 INFO  [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer(976): stopping server 82891a793a29,42251,1732253791157; all regions closed.
2024-11-22T05:37:22,963 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:22,963 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:22,963 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:22,964 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:22,964 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:22,968 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741834_1010 (size=3306)
2024-11-22T05:37:22,969 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741834_1010 (size=3306)
2024-11-22T05:37:22,973 DEBUG [RS:0;82891a793a29:42251 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/oldWALs
2024-11-22T05:37:22,974 INFO  [RS:0;82891a793a29:42251 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 82891a793a29%2C42251%2C1732253791157.meta:.meta(num 1732253792129)
2024-11-22T05:37:22,974 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:22,974 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:22,974 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:22,975 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:22,975 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:22,977 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741844_1020 (size=1252)
2024-11-22T05:37:22,977 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741844_1020 (size=1252)
2024-11-22T05:37:22,982 DEBUG [RS:0;82891a793a29:42251 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/oldWALs
2024-11-22T05:37:22,982 INFO  [RS:0;82891a793a29:42251 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 82891a793a29%2C42251%2C1732253791157:(num 1732253842706)
2024-11-22T05:37:22,982 DEBUG [RS:0;82891a793a29:42251 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:37:22,982 INFO  [RS:0;82891a793a29:42251 {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T05:37:22,982 INFO  [RS:0;82891a793a29:42251 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T05:37:22,982 INFO  [RS:0;82891a793a29:42251 {}] hbase.ChoreService(370): Chore service for: regionserver/82891a793a29:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-22T05:37:22,982 INFO  [RS:0;82891a793a29:42251 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T05:37:22,982 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T05:37:22,982 INFO  [RS:0;82891a793a29:42251 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42251
2024-11-22T05:37:22,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/82891a793a29,42251,1732253791157
2024-11-22T05:37:22,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T05:37:22,991 INFO  [RS:0;82891a793a29:42251 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T05:37:23,002 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [82891a793a29,42251,1732253791157]
2024-11-22T05:37:23,012 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/82891a793a29,42251,1732253791157 already deleted, retry=false
2024-11-22T05:37:23,013 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 82891a793a29,42251,1732253791157 expired; onlineServers=0
2024-11-22T05:37:23,013 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '82891a793a29,44241,1732253790999' *****
2024-11-22T05:37:23,013 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-22T05:37:23,013 INFO  [M:0;82891a793a29:44241 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T05:37:23,013 INFO  [M:0;82891a793a29:44241 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T05:37:23,013 DEBUG [M:0;82891a793a29:44241 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-22T05:37:23,013 DEBUG [M:0;82891a793a29:44241 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-22T05:37:23,013 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-22T05:37:23,013 DEBUG [master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253791498 {}] cleaner.HFileCleaner(306): Exit Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253791498,5,FailOnTimeoutGroup]
2024-11-22T05:37:23,013 DEBUG [master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253791498 {}] cleaner.HFileCleaner(306): Exit Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253791498,5,FailOnTimeoutGroup]
2024-11-22T05:37:23,014 INFO  [M:0;82891a793a29:44241 {}] hbase.ChoreService(370): Chore service for: master/82891a793a29:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-22T05:37:23,014 INFO  [M:0;82891a793a29:44241 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T05:37:23,014 DEBUG [M:0;82891a793a29:44241 {}] master.HMaster(1795): Stopping service threads
2024-11-22T05:37:23,014 INFO  [M:0;82891a793a29:44241 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-22T05:37:23,015 INFO  [M:0;82891a793a29:44241 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T05:37:23,015 INFO  [M:0;82891a793a29:44241 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-22T05:37:23,015 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-22T05:37:23,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-22T05:37:23,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:37:23,023 DEBUG [M:0;82891a793a29:44241 {}] zookeeper.ZKUtil(347): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-22T05:37:23,024 WARN  [M:0;82891a793a29:44241 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-22T05:37:23,024 INFO  [M:0;82891a793a29:44241 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/.lastflushedseqids
2024-11-22T05:37:23,029 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741849_1025 (size=130)
2024-11-22T05:37:23,029 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741849_1025 (size=130)
2024-11-22T05:37:23,030 INFO  [M:0;82891a793a29:44241 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-22T05:37:23,030 INFO  [M:0;82891a793a29:44241 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-22T05:37:23,030 DEBUG [M:0;82891a793a29:44241 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T05:37:23,030 INFO  [M:0;82891a793a29:44241 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:37:23,030 DEBUG [M:0;82891a793a29:44241 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:37:23,030 DEBUG [M:0;82891a793a29:44241 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T05:37:23,030 DEBUG [M:0;82891a793a29:44241 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:37:23,030 INFO  [M:0;82891a793a29:44241 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.55 KB heapSize=54.94 KB
2024-11-22T05:37:23,047 DEBUG [M:0;82891a793a29:44241 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3cc988718a0e4b22a96615870c83be01 is 82, key is hbase:meta,,1/info:regioninfo/1732253792158/Put/seqid=0
2024-11-22T05:37:23,051 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741850_1026 (size=5672)
2024-11-22T05:37:23,052 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741850_1026 (size=5672)
2024-11-22T05:37:23,052 INFO  [M:0;82891a793a29:44241 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3cc988718a0e4b22a96615870c83be01
2024-11-22T05:37:23,070 DEBUG [M:0;82891a793a29:44241 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a9ade70b1c3d41ff8601b404332afcdb is 797, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732253792680/Put/seqid=0
2024-11-22T05:37:23,074 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741851_1027 (size=7819)
2024-11-22T05:37:23,074 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741851_1027 (size=7819)
2024-11-22T05:37:23,074 INFO  [M:0;82891a793a29:44241 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.95 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a9ade70b1c3d41ff8601b404332afcdb
2024-11-22T05:37:23,078 INFO  [M:0;82891a793a29:44241 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a9ade70b1c3d41ff8601b404332afcdb
2024-11-22T05:37:23,092 DEBUG [M:0;82891a793a29:44241 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2b3c2ec20bde499ab249c80202251dc3 is 69, key is 82891a793a29,42251,1732253791157/rs:state/1732253791627/Put/seqid=0
2024-11-22T05:37:23,096 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741852_1028 (size=5156)
2024-11-22T05:37:23,096 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741852_1028 (size=5156)
2024-11-22T05:37:23,097 INFO  [M:0;82891a793a29:44241 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2b3c2ec20bde499ab249c80202251dc3
2024-11-22T05:37:23,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:37:23,102 INFO  [RS:0;82891a793a29:42251 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T05:37:23,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x101610b2eb50001, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:37:23,102 INFO  [RS:0;82891a793a29:42251 {}] regionserver.HRegionServer(1031): Exiting; stopping=82891a793a29,42251,1732253791157; zookeeper connection closed.
2024-11-22T05:37:23,103 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@67acce1a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@67acce1a
2024-11-22T05:37:23,103 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-22T05:37:23,118 DEBUG [M:0;82891a793a29:44241 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a3412edeff9c4c5481808b10b3a7fd3a is 52, key is load_balancer_on/state:d/1732253792293/Put/seqid=0
2024-11-22T05:37:23,125 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741853_1029 (size=5056)
2024-11-22T05:37:23,125 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741853_1029 (size=5056)
2024-11-22T05:37:23,126 INFO  [M:0;82891a793a29:44241 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a3412edeff9c4c5481808b10b3a7fd3a
2024-11-22T05:37:23,130 DEBUG [M:0;82891a793a29:44241 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3cc988718a0e4b22a96615870c83be01 as hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3cc988718a0e4b22a96615870c83be01
2024-11-22T05:37:23,135 INFO  [M:0;82891a793a29:44241 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3cc988718a0e4b22a96615870c83be01, entries=8, sequenceid=121, filesize=5.5 K
2024-11-22T05:37:23,136 DEBUG [M:0;82891a793a29:44241 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a9ade70b1c3d41ff8601b404332afcdb as hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a9ade70b1c3d41ff8601b404332afcdb
2024-11-22T05:37:23,141 INFO  [M:0;82891a793a29:44241 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a9ade70b1c3d41ff8601b404332afcdb
2024-11-22T05:37:23,141 INFO  [M:0;82891a793a29:44241 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a9ade70b1c3d41ff8601b404332afcdb, entries=14, sequenceid=121, filesize=7.6 K
2024-11-22T05:37:23,142 DEBUG [M:0;82891a793a29:44241 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2b3c2ec20bde499ab249c80202251dc3 as hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2b3c2ec20bde499ab249c80202251dc3
2024-11-22T05:37:23,147 INFO  [M:0;82891a793a29:44241 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2b3c2ec20bde499ab249c80202251dc3, entries=1, sequenceid=121, filesize=5.0 K
2024-11-22T05:37:23,148 DEBUG [M:0;82891a793a29:44241 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a3412edeff9c4c5481808b10b3a7fd3a as hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a3412edeff9c4c5481808b10b3a7fd3a
2024-11-22T05:37:23,155 INFO  [M:0;82891a793a29:44241 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33769/user/jenkins/test-data/7979c317-9d84-724d-add7-871d067107d1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a3412edeff9c4c5481808b10b3a7fd3a, entries=1, sequenceid=121, filesize=4.9 K
2024-11-22T05:37:23,156 INFO  [M:0;82891a793a29:44241 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.55 KB/44599, heapSize ~54.88 KB/56192, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 126ms, sequenceid=121, compaction requested=false
2024-11-22T05:37:23,158 INFO  [M:0;82891a793a29:44241 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:37:23,158 DEBUG [M:0;82891a793a29:44241 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732253843030Disabling compacts and flushes for region at 1732253843030Disabling writes for close at 1732253843030Obtaining lock to block concurrent updates at 1732253843030Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732253843030Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44599, getHeapSize=56192, getOffHeapSize=0, getCellsCount=140 at 1732253843030Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732253843031 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732253843031Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732253843047 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732253843047Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732253843056 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732253843069 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732253843069Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732253843078 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732253843092 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732253843092Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732253843101 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732253843118 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732253843118Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d249233: reopening flushed file at 1732253843129 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bade0e: reopening flushed file at 1732253843135 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@467cf0ad: reopening flushed file at 1732253843141 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@790ee328: reopening flushed file at 1732253843147 (+6 ms)Finished flush of dataSize ~43.55 KB/44599, heapSize ~54.88 KB/56192, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 126ms, sequenceid=121, compaction requested=false at 1732253843156 (+9 ms)Writing region close event to WAL at 1732253843157 (+1 ms)Closed at 1732253843157
2024-11-22T05:37:23,158 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:23,158 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:23,158 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:23,158 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:23,158 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:37:23,160 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741830_1006 (size=52996)
2024-11-22T05:37:23,160 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43975 is added to blk_1073741830_1006 (size=52996)
2024-11-22T05:37:23,161 INFO  [M:0;82891a793a29:44241 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-22T05:37:23,161 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T05:37:23,161 INFO  [M:0;82891a793a29:44241 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44241
2024-11-22T05:37:23,161 INFO  [M:0;82891a793a29:44241 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T05:37:23,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:37:23,271 INFO  [M:0;82891a793a29:44241 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T05:37:23,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44241-0x101610b2eb50000, quorum=127.0.0.1:62440, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:37:23,276 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1beefc80{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:37:23,277 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@10c2896a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:37:23,277 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:37:23,277 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@49c1230f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:37:23,277 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47ccf9d3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/hadoop.log.dir/,STOPPED}
2024-11-22T05:37:23,280 WARN  [BP-163586823-172.17.0.2-1732253788519 heartbeating to localhost/127.0.0.1:33769 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:37:23,280 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:37:23,280 WARN  [BP-163586823-172.17.0.2-1732253788519 heartbeating to localhost/127.0.0.1:33769 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-163586823-172.17.0.2-1732253788519 (Datanode Uuid 153ca1ff-2a5b-4914-95db-5b124d576104) service to localhost/127.0.0.1:33769
2024-11-22T05:37:23,280 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:37:23,281 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/cluster_88355450-062b-87e6-5738-3243a3ec2fb3/data/data3/current/BP-163586823-172.17.0.2-1732253788519 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:37:23,281 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/cluster_88355450-062b-87e6-5738-3243a3ec2fb3/data/data4/current/BP-163586823-172.17.0.2-1732253788519 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:37:23,281 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:37:23,284 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3be31a0b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:37:23,284 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2cfed7b4{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:37:23,284 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:37:23,284 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@424c779{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:37:23,284 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b3ac0d9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/hadoop.log.dir/,STOPPED}
2024-11-22T05:37:23,286 WARN  [BP-163586823-172.17.0.2-1732253788519 heartbeating to localhost/127.0.0.1:33769 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:37:23,286 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:37:23,286 WARN  [BP-163586823-172.17.0.2-1732253788519 heartbeating to localhost/127.0.0.1:33769 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-163586823-172.17.0.2-1732253788519 (Datanode Uuid 3757d4af-3767-4c1d-8089-57b6e355f415) service to localhost/127.0.0.1:33769
2024-11-22T05:37:23,286 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:37:23,286 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/cluster_88355450-062b-87e6-5738-3243a3ec2fb3/data/data1/current/BP-163586823-172.17.0.2-1732253788519 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:37:23,287 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/cluster_88355450-062b-87e6-5738-3243a3ec2fb3/data/data2/current/BP-163586823-172.17.0.2-1732253788519 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:37:23,287 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:37:23,292 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6da95783{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T05:37:23,293 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@293e66d4{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:37:23,293 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:37:23,293 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@9cd716d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:37:23,293 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4e960720{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/hadoop.log.dir/,STOPPED}
2024-11-22T05:37:23,299 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-22T05:37:23,316 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-22T05:37:23,323 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=206 (was 180)
Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:33769
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33769 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/82891a793a29:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33769
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33769
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33769 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:33769
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33769
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33769 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=161 (was 221), ProcessCount=11 (was 11), AvailableMemoryMB=7064 (was 7169)
2024-11-22T05:37:23,330 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=206, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=161, ProcessCount=11, AvailableMemoryMB=7064
2024-11-22T05:37:23,330 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-22T05:37:23,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/hadoop.log.dir so I do NOT create it in target/test-data/85e6540c-87cf-4296-b343-d071cebf5323
2024-11-22T05:37:23,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a894e499-40fc-da52-8c74-cdf2f9f72ded/hadoop.tmp.dir so I do NOT create it in target/test-data/85e6540c-87cf-4296-b343-d071cebf5323
2024-11-22T05:37:23,331 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/cluster_48c49796-9abf-9856-dddf-c5191332e906, deleteOnExit=true
2024-11-22T05:37:23,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-22T05:37:23,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/test.cache.data in system properties and HBase conf
2024-11-22T05:37:23,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/hadoop.tmp.dir in system properties and HBase conf
2024-11-22T05:37:23,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/hadoop.log.dir in system properties and HBase conf
2024-11-22T05:37:23,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-22T05:37:23,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-22T05:37:23,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-22T05:37:23,331 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-22T05:37:23,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-22T05:37:23,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-22T05:37:23,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-22T05:37:23,332 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T05:37:23,332 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-22T05:37:23,332 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-22T05:37:23,332 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T05:37:23,332 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T05:37:23,332 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-22T05:37:23,332 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/nfs.dump.dir in system properties and HBase conf
2024-11-22T05:37:23,332 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/java.io.tmpdir in system properties and HBase conf
2024-11-22T05:37:23,332 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T05:37:23,332 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-22T05:37:23,332 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-22T05:37:23,345 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T05:37:23,650 INFO  [regionserver/82891a793a29:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T05:37:23,727 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:23,727 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:23,736 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:37:23,740 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:37:23,741 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:37:23,741 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:37:23,741 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T05:37:23,741 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:37:23,742 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@79864455{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:37:23,742 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@374d3611{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:37:23,841 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@671b15e6{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/java.io.tmpdir/jetty-localhost-37621-hadoop-hdfs-3_4_1-tests_jar-_-any-3186174669713704330/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T05:37:23,842 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2996c87f{HTTP/1.1, (http/1.1)}{localhost:37621}
2024-11-22T05:37:23,842 INFO  [Time-limited test {}] server.Server(415): Started @251099ms
2024-11-22T05:37:23,852 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T05:37:24,163 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:37:24,165 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:37:24,172 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:37:24,172 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:37:24,172 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:37:24,173 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@354bdaa4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:37:24,173 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@18d1ee92{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:37:24,266 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@23e1642c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/java.io.tmpdir/jetty-localhost-34685-hadoop-hdfs-3_4_1-tests_jar-_-any-13238904713726583251/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:37:24,266 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@767f877d{HTTP/1.1, (http/1.1)}{localhost:34685}
2024-11-22T05:37:24,266 INFO  [Time-limited test {}] server.Server(415): Started @251524ms
2024-11-22T05:37:24,267 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:37:24,296 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:37:24,300 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:37:24,301 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:37:24,301 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:37:24,301 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T05:37:24,302 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49490ce4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:37:24,302 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@19fe8881{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:37:24,394 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@47bcda8c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/java.io.tmpdir/jetty-localhost-45155-hadoop-hdfs-3_4_1-tests_jar-_-any-15760980820513138072/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:37:24,395 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@29765213{HTTP/1.1, (http/1.1)}{localhost:45155}
2024-11-22T05:37:24,395 INFO  [Time-limited test {}] server.Server(415): Started @251652ms
2024-11-22T05:37:24,396 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:37:24,728 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:24,728 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:25,729 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:25,729 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:25,777 WARN  [Thread-1979 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/cluster_48c49796-9abf-9856-dddf-c5191332e906/data/data1/current/BP-1931242728-172.17.0.2-1732253843348/current, will proceed with Du for space computation calculation, 
2024-11-22T05:37:25,777 WARN  [Thread-1980 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/cluster_48c49796-9abf-9856-dddf-c5191332e906/data/data2/current/BP-1931242728-172.17.0.2-1732253843348/current, will proceed with Du for space computation calculation, 
2024-11-22T05:37:25,795 WARN  [Thread-1943 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:37:25,797 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x17a1d7d0ed061bbf with lease ID 0x2bdcf542c903f3ae: Processing first storage report for DS-b8d8c3af-5c47-4e4a-a877-c0f0e0b4affd from datanode DatanodeRegistration(127.0.0.1:39999, datanodeUuid=49d89928-7cc0-4701-83b0-5817393f9506, infoPort=37343, infoSecurePort=0, ipcPort=34821, storageInfo=lv=-57;cid=testClusterID;nsid=1819259160;c=1732253843348)
2024-11-22T05:37:25,797 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x17a1d7d0ed061bbf with lease ID 0x2bdcf542c903f3ae: from storage DS-b8d8c3af-5c47-4e4a-a877-c0f0e0b4affd node DatanodeRegistration(127.0.0.1:39999, datanodeUuid=49d89928-7cc0-4701-83b0-5817393f9506, infoPort=37343, infoSecurePort=0, ipcPort=34821, storageInfo=lv=-57;cid=testClusterID;nsid=1819259160;c=1732253843348), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:37:25,797 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x17a1d7d0ed061bbf with lease ID 0x2bdcf542c903f3ae: Processing first storage report for DS-c0b7c2ca-e259-4ba2-9b77-906dca4e2c63 from datanode DatanodeRegistration(127.0.0.1:39999, datanodeUuid=49d89928-7cc0-4701-83b0-5817393f9506, infoPort=37343, infoSecurePort=0, ipcPort=34821, storageInfo=lv=-57;cid=testClusterID;nsid=1819259160;c=1732253843348)
2024-11-22T05:37:25,797 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x17a1d7d0ed061bbf with lease ID 0x2bdcf542c903f3ae: from storage DS-c0b7c2ca-e259-4ba2-9b77-906dca4e2c63 node DatanodeRegistration(127.0.0.1:39999, datanodeUuid=49d89928-7cc0-4701-83b0-5817393f9506, infoPort=37343, infoSecurePort=0, ipcPort=34821, storageInfo=lv=-57;cid=testClusterID;nsid=1819259160;c=1732253843348), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:37:25,930 WARN  [Thread-1990 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/cluster_48c49796-9abf-9856-dddf-c5191332e906/data/data3/current/BP-1931242728-172.17.0.2-1732253843348/current, will proceed with Du for space computation calculation, 
2024-11-22T05:37:25,931 WARN  [Thread-1991 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/cluster_48c49796-9abf-9856-dddf-c5191332e906/data/data4/current/BP-1931242728-172.17.0.2-1732253843348/current, will proceed with Du for space computation calculation, 
2024-11-22T05:37:25,950 WARN  [Thread-1966 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:37:25,951 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xda4c20617dba01c3 with lease ID 0x2bdcf542c903f3af: Processing first storage report for DS-76a2b62c-1855-4152-b1de-e13a2bc8c0f7 from datanode DatanodeRegistration(127.0.0.1:35043, datanodeUuid=d73574e7-bd8c-462d-9854-b1281436546b, infoPort=46427, infoSecurePort=0, ipcPort=36269, storageInfo=lv=-57;cid=testClusterID;nsid=1819259160;c=1732253843348)
2024-11-22T05:37:25,951 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda4c20617dba01c3 with lease ID 0x2bdcf542c903f3af: from storage DS-76a2b62c-1855-4152-b1de-e13a2bc8c0f7 node DatanodeRegistration(127.0.0.1:35043, datanodeUuid=d73574e7-bd8c-462d-9854-b1281436546b, infoPort=46427, infoSecurePort=0, ipcPort=36269, storageInfo=lv=-57;cid=testClusterID;nsid=1819259160;c=1732253843348), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:37:25,952 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xda4c20617dba01c3 with lease ID 0x2bdcf542c903f3af: Processing first storage report for DS-c82924a7-0aac-4890-bfd2-b486c7517d6f from datanode DatanodeRegistration(127.0.0.1:35043, datanodeUuid=d73574e7-bd8c-462d-9854-b1281436546b, infoPort=46427, infoSecurePort=0, ipcPort=36269, storageInfo=lv=-57;cid=testClusterID;nsid=1819259160;c=1732253843348)
2024-11-22T05:37:25,952 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda4c20617dba01c3 with lease ID 0x2bdcf542c903f3af: from storage DS-c82924a7-0aac-4890-bfd2-b486c7517d6f node DatanodeRegistration(127.0.0.1:35043, datanodeUuid=d73574e7-bd8c-462d-9854-b1281436546b, infoPort=46427, infoSecurePort=0, ipcPort=36269, storageInfo=lv=-57;cid=testClusterID;nsid=1819259160;c=1732253843348), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:37:26,028 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323
2024-11-22T05:37:26,032 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/cluster_48c49796-9abf-9856-dddf-c5191332e906/zookeeper_0, clientPort=57233, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/cluster_48c49796-9abf-9856-dddf-c5191332e906/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/cluster_48c49796-9abf-9856-dddf-c5191332e906/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-22T05:37:26,033 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57233
2024-11-22T05:37:26,033 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:37:26,035 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:37:26,042 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741825_1001 (size=7)
2024-11-22T05:37:26,042 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741825_1001 (size=7)
2024-11-22T05:37:26,043 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da with version=8
2024-11-22T05:37:26,043 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/hbase-staging
2024-11-22T05:37:26,045 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/82891a793a29:0 server-side Connection retries=45
2024-11-22T05:37:26,045 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:37:26,045 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T05:37:26,045 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T05:37:26,045 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:37:26,045 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T05:37:26,045 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-22T05:37:26,045 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T05:37:26,046 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44627
2024-11-22T05:37:26,047 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44627 connecting to ZooKeeper ensemble=127.0.0.1:57233
2024-11-22T05:37:26,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:446270x0, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T05:37:26,105 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44627-0x101610c05ba0000 connected
2024-11-22T05:37:26,200 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:37:26,202 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:37:26,205 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:37:26,205 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da, hbase.cluster.distributed=false
2024-11-22T05:37:26,207 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T05:37:26,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44627
2024-11-22T05:37:26,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44627
2024-11-22T05:37:26,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44627
2024-11-22T05:37:26,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44627
2024-11-22T05:37:26,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44627
2024-11-22T05:37:26,223 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/82891a793a29:0 server-side Connection retries=45
2024-11-22T05:37:26,223 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:37:26,223 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T05:37:26,223 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T05:37:26,223 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:37:26,223 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T05:37:26,223 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-22T05:37:26,223 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T05:37:26,224 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42063
2024-11-22T05:37:26,225 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42063 connecting to ZooKeeper ensemble=127.0.0.1:57233
2024-11-22T05:37:26,225 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:37:26,227 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:37:26,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:420630x0, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T05:37:26,236 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:37:26,236 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42063-0x101610c05ba0001 connected
2024-11-22T05:37:26,236 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-22T05:37:26,237 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-22T05:37:26,237 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-22T05:37:26,238 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T05:37:26,239 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42063
2024-11-22T05:37:26,239 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42063
2024-11-22T05:37:26,239 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42063
2024-11-22T05:37:26,240 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42063
2024-11-22T05:37:26,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42063
2024-11-22T05:37:26,254 DEBUG [M:0;82891a793a29:44627 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;82891a793a29:44627
2024-11-22T05:37:26,255 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/82891a793a29,44627,1732253846045
2024-11-22T05:37:26,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:37:26,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:37:26,268 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/82891a793a29,44627,1732253846045
2024-11-22T05:37:26,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-22T05:37:26,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:37:26,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:37:26,278 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-22T05:37:26,278 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/82891a793a29,44627,1732253846045 from backup master directory
2024-11-22T05:37:26,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:37:26,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/82891a793a29,44627,1732253846045
2024-11-22T05:37:26,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:37:26,288 WARN  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T05:37:26,288 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=82891a793a29,44627,1732253846045
2024-11-22T05:37:26,292 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/hbase.id] with ID: 33991321-79e4-46f5-b91b-8982dff16362
2024-11-22T05:37:26,292 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/.tmp/hbase.id
2024-11-22T05:37:26,302 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741826_1002 (size=42)
2024-11-22T05:37:26,302 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741826_1002 (size=42)
2024-11-22T05:37:26,303 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/.tmp/hbase.id]:[hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/hbase.id]
2024-11-22T05:37:26,313 INFO  [master/82891a793a29:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:37:26,313 INFO  [master/82891a793a29:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-22T05:37:26,314 INFO  [master/82891a793a29:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-22T05:37:26,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:37:26,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:37:26,331 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741827_1003 (size=196)
2024-11-22T05:37:26,332 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741827_1003 (size=196)
2024-11-22T05:37:26,332 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T05:37:26,334 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-22T05:37:26,334 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:37:26,341 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741828_1004 (size=1189)
2024-11-22T05:37:26,341 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741828_1004 (size=1189)
2024-11-22T05:37:26,342 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store
2024-11-22T05:37:26,348 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741829_1005 (size=34)
2024-11-22T05:37:26,348 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741829_1005 (size=34)
2024-11-22T05:37:26,349 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:37:26,349 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T05:37:26,349 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:37:26,349 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:37:26,349 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T05:37:26,349 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:37:26,349 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:37:26,349 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732253846349Disabling compacts and flushes for region at 1732253846349Disabling writes for close at 1732253846349Writing region close event to WAL at 1732253846349Closed at 1732253846349
2024-11-22T05:37:26,350 WARN  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/.initializing
2024-11-22T05:37:26,350 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/WALs/82891a793a29,44627,1732253846045
2024-11-22T05:37:26,352 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C44627%2C1732253846045, suffix=, logDir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/WALs/82891a793a29,44627,1732253846045, archiveDir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/oldWALs, maxLogs=10
2024-11-22T05:37:26,353 INFO  [master/82891a793a29:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C44627%2C1732253846045.1732253846353
2024-11-22T05:37:26,360 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/WALs/82891a793a29,44627,1732253846045/82891a793a29%2C44627%2C1732253846045.1732253846353
2024-11-22T05:37:26,362 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37343:37343),(127.0.0.1/127.0.0.1:46427:46427)]
2024-11-22T05:37:26,364 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:37:26,364 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:37:26,364 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:37:26,364 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:37:26,368 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:37:26,370 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-22T05:37:26,370 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:26,371 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:37:26,371 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:37:26,372 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-22T05:37:26,372 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:26,373 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:37:26,373 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:37:26,374 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-22T05:37:26,374 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:26,375 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:37:26,375 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:37:26,376 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-22T05:37:26,376 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:26,376 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:37:26,377 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:37:26,377 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:37:26,378 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:37:26,379 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:37:26,379 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:37:26,380 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-22T05:37:26,381 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:37:26,384 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:37:26,384 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=882268, jitterRate=0.12186205387115479}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-22T05:37:26,385 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732253846364Initializing all the Stores at 1732253846365 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253846365Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253846368 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253846368Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253846368Cleaning up temporary data from old regions at 1732253846379 (+11 ms)Region opened successfully at 1732253846385 (+6 ms)
2024-11-22T05:37:26,385 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-22T05:37:26,389 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e49b67f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82891a793a29/172.17.0.2:0
2024-11-22T05:37:26,391 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-22T05:37:26,391 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-22T05:37:26,391 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-22T05:37:26,391 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-22T05:37:26,392 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-22T05:37:26,393 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-22T05:37:26,393 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-22T05:37:26,395 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-22T05:37:26,397 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-22T05:37:26,404 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-22T05:37:26,405 INFO  [master/82891a793a29:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-22T05:37:26,406 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-22T05:37:26,414 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-22T05:37:26,415 INFO  [master/82891a793a29:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-22T05:37:26,417 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-22T05:37:26,425 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-22T05:37:26,426 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-22T05:37:26,435 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-22T05:37:26,438 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-22T05:37:26,446 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-22T05:37:26,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T05:37:26,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T05:37:26,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:37:26,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:37:26,457 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=82891a793a29,44627,1732253846045, sessionid=0x101610c05ba0000, setting cluster-up flag (Was=false)
2024-11-22T05:37:26,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:37:26,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:37:26,509 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-22T05:37:26,511 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=82891a793a29,44627,1732253846045
2024-11-22T05:37:26,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:37:26,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:37:26,562 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-22T05:37:26,563 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=82891a793a29,44627,1732253846045
2024-11-22T05:37:26,565 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-22T05:37:26,566 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-22T05:37:26,567 INFO  [master/82891a793a29:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-22T05:37:26,567 INFO  [master/82891a793a29:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-22T05:37:26,567 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 82891a793a29,44627,1732253846045
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-22T05:37:26,568 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:37:26,568 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:37:26,569 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:37:26,569 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:37:26,569 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/82891a793a29:0, corePoolSize=10, maxPoolSize=10
2024-11-22T05:37:26,569 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:37:26,569 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/82891a793a29:0, corePoolSize=2, maxPoolSize=2
2024-11-22T05:37:26,569 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:37:26,570 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732253876570
2024-11-22T05:37:26,570 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-22T05:37:26,570 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-22T05:37:26,570 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-22T05:37:26,570 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-22T05:37:26,570 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-22T05:37:26,570 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-22T05:37:26,571 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:26,571 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-22T05:37:26,571 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-22T05:37:26,571 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-22T05:37:26,571 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:37:26,571 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-22T05:37:26,571 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-22T05:37:26,571 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-22T05:37:26,572 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253846571,5,FailOnTimeoutGroup]
2024-11-22T05:37:26,572 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:26,573 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-22T05:37:26,574 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253846572,5,FailOnTimeoutGroup]
2024-11-22T05:37:26,574 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:26,574 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-22T05:37:26,574 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:26,574 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:26,579 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741831_1007 (size=1321)
2024-11-22T05:37:26,580 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741831_1007 (size=1321)
2024-11-22T05:37:26,580 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-22T05:37:26,580 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da
2024-11-22T05:37:26,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741832_1008 (size=32)
2024-11-22T05:37:26,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741832_1008 (size=32)
2024-11-22T05:37:26,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:37:26,588 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T05:37:26,589 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T05:37:26,589 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:26,590 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:37:26,590 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T05:37:26,591 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T05:37:26,591 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:26,591 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:37:26,591 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T05:37:26,592 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T05:37:26,592 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:26,593 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:37:26,593 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T05:37:26,594 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T05:37:26,594 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:26,594 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:37:26,594 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T05:37:26,595 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740
2024-11-22T05:37:26,595 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740
2024-11-22T05:37:26,597 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T05:37:26,597 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T05:37:26,597 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T05:37:26,598 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T05:37:26,600 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:37:26,601 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=741415, jitterRate=-0.05724330246448517}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T05:37:26,602 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732253846587Initializing all the Stores at 1732253846588 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253846588Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253846588Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253846588Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253846588Cleaning up temporary data from old regions at 1732253846597 (+9 ms)Region opened successfully at 1732253846602 (+5 ms)
2024-11-22T05:37:26,602 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T05:37:26,602 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T05:37:26,602 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T05:37:26,602 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T05:37:26,602 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T05:37:26,602 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T05:37:26,602 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732253846602Disabling compacts and flushes for region at 1732253846602Disabling writes for close at 1732253846602Writing region close event to WAL at 1732253846602Closed at 1732253846602
2024-11-22T05:37:26,604 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:37:26,604 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-22T05:37:26,604 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-22T05:37:26,605 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T05:37:26,606 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-22T05:37:26,643 INFO  [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(746): ClusterId : 33991321-79e4-46f5-b91b-8982dff16362
2024-11-22T05:37:26,643 DEBUG [RS:0;82891a793a29:42063 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-22T05:37:26,653 DEBUG [RS:0;82891a793a29:42063 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-22T05:37:26,653 DEBUG [RS:0;82891a793a29:42063 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-22T05:37:26,664 DEBUG [RS:0;82891a793a29:42063 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-22T05:37:26,664 DEBUG [RS:0;82891a793a29:42063 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c39e47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82891a793a29/172.17.0.2:0
2024-11-22T05:37:26,681 DEBUG [RS:0;82891a793a29:42063 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;82891a793a29:42063
2024-11-22T05:37:26,681 INFO  [RS:0;82891a793a29:42063 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-22T05:37:26,681 INFO  [RS:0;82891a793a29:42063 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-22T05:37:26,681 DEBUG [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-22T05:37:26,685 INFO  [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(2659): reportForDuty to master=82891a793a29,44627,1732253846045 with port=42063, startcode=1732253846222
2024-11-22T05:37:26,685 DEBUG [RS:0;82891a793a29:42063 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-22T05:37:26,688 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46871, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-11-22T05:37:26,689 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44627 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 82891a793a29,42063,1732253846222
2024-11-22T05:37:26,689 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44627 {}] master.ServerManager(517): Registering regionserver=82891a793a29,42063,1732253846222
2024-11-22T05:37:26,692 DEBUG [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da
2024-11-22T05:37:26,692 DEBUG [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41245
2024-11-22T05:37:26,692 DEBUG [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-22T05:37:26,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T05:37:26,705 DEBUG [RS:0;82891a793a29:42063 {}] zookeeper.ZKUtil(111): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/82891a793a29,42063,1732253846222
2024-11-22T05:37:26,705 WARN  [RS:0;82891a793a29:42063 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T05:37:26,706 INFO  [RS:0;82891a793a29:42063 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:37:26,706 DEBUG [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/WALs/82891a793a29,42063,1732253846222
2024-11-22T05:37:26,706 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [82891a793a29,42063,1732253846222]
2024-11-22T05:37:26,709 INFO  [RS:0;82891a793a29:42063 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-22T05:37:26,712 INFO  [RS:0;82891a793a29:42063 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-22T05:37:26,712 INFO  [RS:0;82891a793a29:42063 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T05:37:26,712 INFO  [RS:0;82891a793a29:42063 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:26,712 INFO  [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-22T05:37:26,713 INFO  [RS:0;82891a793a29:42063 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-22T05:37:26,713 INFO  [RS:0;82891a793a29:42063 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:26,713 DEBUG [RS:0;82891a793a29:42063 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:37:26,713 DEBUG [RS:0;82891a793a29:42063 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:37:26,713 DEBUG [RS:0;82891a793a29:42063 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:37:26,714 DEBUG [RS:0;82891a793a29:42063 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:37:26,714 DEBUG [RS:0;82891a793a29:42063 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:37:26,714 DEBUG [RS:0;82891a793a29:42063 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/82891a793a29:0, corePoolSize=2, maxPoolSize=2
2024-11-22T05:37:26,714 DEBUG [RS:0;82891a793a29:42063 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:37:26,714 DEBUG [RS:0;82891a793a29:42063 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:37:26,714 DEBUG [RS:0;82891a793a29:42063 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:37:26,714 DEBUG [RS:0;82891a793a29:42063 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:37:26,714 DEBUG [RS:0;82891a793a29:42063 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:37:26,714 DEBUG [RS:0;82891a793a29:42063 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:37:26,714 DEBUG [RS:0;82891a793a29:42063 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/82891a793a29:0, corePoolSize=3, maxPoolSize=3
2024-11-22T05:37:26,714 DEBUG [RS:0;82891a793a29:42063 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0, corePoolSize=3, maxPoolSize=3
2024-11-22T05:37:26,717 INFO  [RS:0;82891a793a29:42063 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:26,717 INFO  [RS:0;82891a793a29:42063 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:26,717 INFO  [RS:0;82891a793a29:42063 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:26,717 INFO  [RS:0;82891a793a29:42063 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:26,717 INFO  [RS:0;82891a793a29:42063 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:26,717 INFO  [RS:0;82891a793a29:42063 {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,42063,1732253846222-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T05:37:26,730 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:26,730 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:26,735 INFO  [RS:0;82891a793a29:42063 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-22T05:37:26,735 INFO  [RS:0;82891a793a29:42063 {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,42063,1732253846222-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:26,736 INFO  [RS:0;82891a793a29:42063 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:26,736 INFO  [RS:0;82891a793a29:42063 {}] regionserver.Replication(171): 82891a793a29,42063,1732253846222 started
2024-11-22T05:37:26,750 INFO  [RS:0;82891a793a29:42063 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:26,750 INFO  [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(1482): Serving as 82891a793a29,42063,1732253846222, RpcServer on 82891a793a29/172.17.0.2:42063, sessionid=0x101610c05ba0001
2024-11-22T05:37:26,751 DEBUG [RS:0;82891a793a29:42063 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-22T05:37:26,751 DEBUG [RS:0;82891a793a29:42063 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 82891a793a29,42063,1732253846222
2024-11-22T05:37:26,751 DEBUG [RS:0;82891a793a29:42063 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82891a793a29,42063,1732253846222'
2024-11-22T05:37:26,751 DEBUG [RS:0;82891a793a29:42063 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-22T05:37:26,751 DEBUG [RS:0;82891a793a29:42063 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-22T05:37:26,752 DEBUG [RS:0;82891a793a29:42063 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-22T05:37:26,752 DEBUG [RS:0;82891a793a29:42063 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-22T05:37:26,752 DEBUG [RS:0;82891a793a29:42063 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 82891a793a29,42063,1732253846222
2024-11-22T05:37:26,752 DEBUG [RS:0;82891a793a29:42063 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82891a793a29,42063,1732253846222'
2024-11-22T05:37:26,752 DEBUG [RS:0;82891a793a29:42063 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-22T05:37:26,752 DEBUG [RS:0;82891a793a29:42063 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-22T05:37:26,752 DEBUG [RS:0;82891a793a29:42063 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-22T05:37:26,752 INFO  [RS:0;82891a793a29:42063 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-22T05:37:26,752 INFO  [RS:0;82891a793a29:42063 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-22T05:37:26,757 WARN  [82891a793a29:44627 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-22T05:37:26,854 INFO  [RS:0;82891a793a29:42063 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C42063%2C1732253846222, suffix=, logDir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/WALs/82891a793a29,42063,1732253846222, archiveDir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/oldWALs, maxLogs=32
2024-11-22T05:37:26,855 INFO  [RS:0;82891a793a29:42063 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42063%2C1732253846222.1732253846855
2024-11-22T05:37:26,861 INFO  [RS:0;82891a793a29:42063 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/WALs/82891a793a29,42063,1732253846222/82891a793a29%2C42063%2C1732253846222.1732253846855
2024-11-22T05:37:26,868 DEBUG [RS:0;82891a793a29:42063 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46427:46427),(127.0.0.1/127.0.0.1:37343:37343)]
2024-11-22T05:37:27,007 DEBUG [82891a793a29:44627 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-22T05:37:27,007 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=82891a793a29,42063,1732253846222
2024-11-22T05:37:27,009 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 82891a793a29,42063,1732253846222, state=OPENING
2024-11-22T05:37:27,097 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-22T05:37:27,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:37:27,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:37:27,170 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:37:27,170 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:37:27,170 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T05:37:27,170 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=82891a793a29,42063,1732253846222}]
2024-11-22T05:37:27,324 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-22T05:37:27,325 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54273, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-22T05:37:27,332 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-22T05:37:27,332 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:37:27,334 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C42063%2C1732253846222.meta, suffix=.meta, logDir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/WALs/82891a793a29,42063,1732253846222, archiveDir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/oldWALs, maxLogs=32
2024-11-22T05:37:27,334 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42063%2C1732253846222.meta.1732253847334.meta
2024-11-22T05:37:27,341 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/WALs/82891a793a29,42063,1732253846222/82891a793a29%2C42063%2C1732253846222.meta.1732253847334.meta
2024-11-22T05:37:27,353 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46427:46427),(127.0.0.1/127.0.0.1:37343:37343)]
2024-11-22T05:37:27,356 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:37:27,356 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-22T05:37:27,356 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-22T05:37:27,357 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-22T05:37:27,357 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-22T05:37:27,357 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:37:27,357 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-22T05:37:27,357 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-22T05:37:27,360 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T05:37:27,361 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T05:37:27,361 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:27,362 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:37:27,362 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T05:37:27,363 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T05:37:27,364 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:27,364 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:37:27,364 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T05:37:27,365 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T05:37:27,365 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:27,366 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:37:27,366 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T05:37:27,367 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T05:37:27,367 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:27,367 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:37:27,368 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T05:37:27,369 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740
2024-11-22T05:37:27,370 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740
2024-11-22T05:37:27,372 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T05:37:27,372 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T05:37:27,372 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T05:37:27,375 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T05:37:27,375 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=711690, jitterRate=-0.09503954648971558}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T05:37:27,376 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-22T05:37:27,376 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732253847357Writing region info on filesystem at 1732253847357Initializing all the Stores at 1732253847358 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253847358Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253847360 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253847360Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253847360Cleaning up temporary data from old regions at 1732253847372 (+12 ms)Running coprocessor post-open hooks at 1732253847376 (+4 ms)Region opened successfully at 1732253847376
2024-11-22T05:37:27,378 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732253847323
2024-11-22T05:37:27,382 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-22T05:37:27,382 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-22T05:37:27,383 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=82891a793a29,42063,1732253846222
2024-11-22T05:37:27,384 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 82891a793a29,42063,1732253846222, state=OPEN
2024-11-22T05:37:27,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T05:37:27,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T05:37:27,484 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=82891a793a29,42063,1732253846222
2024-11-22T05:37:27,484 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:37:27,484 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:37:27,488 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-22T05:37:27,488 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=82891a793a29,42063,1732253846222 in 314 msec
2024-11-22T05:37:27,491 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-22T05:37:27,491 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 884 msec
2024-11-22T05:37:27,492 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:37:27,492 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-22T05:37:27,494 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T05:37:27,494 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82891a793a29,42063,1732253846222, seqNum=-1]
2024-11-22T05:37:27,494 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T05:37:27,496 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48455, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T05:37:27,502 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 935 msec
2024-11-22T05:37:27,502 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732253847502, completionTime=-1
2024-11-22T05:37:27,503 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-22T05:37:27,503 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-22T05:37:27,505 INFO  [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-22T05:37:27,505 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732253907505
2024-11-22T05:37:27,505 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732253967505
2024-11-22T05:37:27,505 INFO  [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-22T05:37:27,506 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,44627,1732253846045-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:27,506 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,44627,1732253846045-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:27,506 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,44627,1732253846045-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:27,506 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-82891a793a29:44627, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:27,506 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:27,506 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:27,508 DEBUG [master/82891a793a29:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-22T05:37:27,517 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.229sec
2024-11-22T05:37:27,518 INFO  [master/82891a793a29:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-22T05:37:27,518 INFO  [master/82891a793a29:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-22T05:37:27,518 INFO  [master/82891a793a29:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-22T05:37:27,518 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-22T05:37:27,518 INFO  [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-22T05:37:27,518 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,44627,1732253846045-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T05:37:27,518 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,44627,1732253846045-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-22T05:37:27,520 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-22T05:37:27,520 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-22T05:37:27,520 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,44627,1732253846045-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:37:27,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc27ae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:37:27,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82891a793a29,44627,-1 for getting cluster id
2024-11-22T05:37:27,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-22T05:37:27,545 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '33991321-79e4-46f5-b91b-8982dff16362'
2024-11-22T05:37:27,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-22T05:37:27,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "33991321-79e4-46f5-b91b-8982dff16362"

2024-11-22T05:37:27,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f97e06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:37:27,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82891a793a29,44627,-1]
2024-11-22T05:37:27,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-22T05:37:27,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:37:27,547 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-22T05:37:27,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b74706c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:37:27,549 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T05:37:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82891a793a29,42063,1732253846222, seqNum=-1]
2024-11-22T05:37:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T05:37:27,551 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T05:37:27,553 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=82891a793a29,44627,1732253846045
2024-11-22T05:37:27,553 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:37:27,556 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-22T05:37:27,556 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-22T05:37:27,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 82891a793a29,44627,1732253846045
2024-11-22T05:37:27,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1504dcab
2024-11-22T05:37:27,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-22T05:37:27,560 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-22T05:37:27,560 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44627 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-22T05:37:27,561 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44627 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-22T05:37:27,561 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44627 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T05:37:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44627 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-11-22T05:37:27,563 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-22T05:37:27,563 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:27,563 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44627 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-11-22T05:37:27,564 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-22T05:37:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T05:37:27,578 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741835_1011 (size=381)
2024-11-22T05:37:27,578 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741835_1011 (size=381)
2024-11-22T05:37:27,581 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c3b99209a6d4a5122f7ec5ad421b3c59, NAME => 'TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da
2024-11-22T05:37:27,590 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741836_1012 (size=64)
2024-11-22T05:37:27,590 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741836_1012 (size=64)
2024-11-22T05:37:27,591 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:37:27,591 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing c3b99209a6d4a5122f7ec5ad421b3c59, disabling compactions & flushes
2024-11-22T05:37:27,591 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.
2024-11-22T05:37:27,591 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.
2024-11-22T05:37:27,591 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59. after waiting 0 ms
2024-11-22T05:37:27,591 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.
2024-11-22T05:37:27,591 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.
2024-11-22T05:37:27,591 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for c3b99209a6d4a5122f7ec5ad421b3c59:
Waiting for close lock at 1732253847591Disabling compacts and flushes for region at 1732253847591Disabling writes for close at 1732253847591Writing region close event to WAL at 1732253847591Closed at 1732253847591
2024-11-22T05:37:27,593 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-22T05:37:27,593 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732253847593"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732253847593"}]},"ts":"1732253847593"}
2024-11-22T05:37:27,596 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-22T05:37:27,598 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-22T05:37:27,598 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732253847598"}]},"ts":"1732253847598"}
2024-11-22T05:37:27,601 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-11-22T05:37:27,601 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c3b99209a6d4a5122f7ec5ad421b3c59, ASSIGN}]
2024-11-22T05:37:27,603 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c3b99209a6d4a5122f7ec5ad421b3c59, ASSIGN
2024-11-22T05:37:27,604 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c3b99209a6d4a5122f7ec5ad421b3c59, ASSIGN; state=OFFLINE, location=82891a793a29,42063,1732253846222; forceNewPlan=false, retain=false
2024-11-22T05:37:27,731 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:27,731 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:27,755 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c3b99209a6d4a5122f7ec5ad421b3c59, regionState=OPENING, regionLocation=82891a793a29,42063,1732253846222
2024-11-22T05:37:27,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c3b99209a6d4a5122f7ec5ad421b3c59, ASSIGN because future has completed
2024-11-22T05:37:27,760 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b99209a6d4a5122f7ec5ad421b3c59, server=82891a793a29,42063,1732253846222}]
2024-11-22T05:37:27,799 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,799 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,800 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,800 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,800 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,800 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,801 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,801 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,832 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,832 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,832 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,833 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,833 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,833 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,839 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,840 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,840 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,843 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:27,917 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.
2024-11-22T05:37:27,917 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c3b99209a6d4a5122f7ec5ad421b3c59, NAME => 'TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:37:27,917 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:27,917 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:37:27,918 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:27,918 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:27,919 INFO  [StoreOpener-c3b99209a6d4a5122f7ec5ad421b3c59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c3b99209a6d4a5122f7ec5ad421b3c59 
2024-11-22T05:37:27,920 INFO  [StoreOpener-c3b99209a6d4a5122f7ec5ad421b3c59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3b99209a6d4a5122f7ec5ad421b3c59 columnFamilyName info
2024-11-22T05:37:27,920 DEBUG [StoreOpener-c3b99209a6d4a5122f7ec5ad421b3c59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:27,921 INFO  [StoreOpener-c3b99209a6d4a5122f7ec5ad421b3c59-1 {}] regionserver.HStore(327): Store=c3b99209a6d4a5122f7ec5ad421b3c59/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:37:27,921 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:27,921 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:27,922 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:27,922 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:27,922 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:27,924 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:27,926 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:37:27,926 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c3b99209a6d4a5122f7ec5ad421b3c59; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=801135, jitterRate=0.018696770071983337}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-22T05:37:27,926 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:27,927 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c3b99209a6d4a5122f7ec5ad421b3c59:
Running coprocessor pre-open hook at 1732253847918Writing region info on filesystem at 1732253847918Initializing all the Stores at 1732253847918Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253847918Cleaning up temporary data from old regions at 1732253847922 (+4 ms)Running coprocessor post-open hooks at 1732253847926 (+4 ms)Region opened successfully at 1732253847927 (+1 ms)
2024-11-22T05:37:27,928 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59., pid=6, masterSystemTime=1732253847913
2024-11-22T05:37:27,931 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.
2024-11-22T05:37:27,931 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.
2024-11-22T05:37:27,932 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c3b99209a6d4a5122f7ec5ad421b3c59, regionState=OPEN, openSeqNum=2, regionLocation=82891a793a29,42063,1732253846222
2024-11-22T05:37:27,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3b99209a6d4a5122f7ec5ad421b3c59, server=82891a793a29,42063,1732253846222 because future has completed
2024-11-22T05:37:27,938 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-22T05:37:27,938 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c3b99209a6d4a5122f7ec5ad421b3c59, server=82891a793a29,42063,1732253846222 in 175 msec
2024-11-22T05:37:27,940 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-22T05:37:27,941 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c3b99209a6d4a5122f7ec5ad421b3c59, ASSIGN in 337 msec
2024-11-22T05:37:27,942 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-22T05:37:27,942 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732253847942"}]},"ts":"1732253847942"}
2024-11-22T05:37:27,944 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-11-22T05:37:27,945 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-22T05:37:27,947 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 384 msec
2024-11-22T05:37:28,350 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T05:37:28,351 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,351 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,351 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,352 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,352 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,352 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,352 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,353 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,378 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,378 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,378 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,381 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:28,731 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:28,731 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:29,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-22T05:37:29,491 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-22T05:37:29,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T05:37:29,732 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:29,732 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:30,733 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:30,733 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:31,734 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:31,734 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:32,709 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-22T05:37:32,710 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-11-22T05:37:32,735 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:32,735 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:33,420 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T05:37:33,420 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,421 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,421 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,421 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,421 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,421 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,422 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,422 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,459 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,460 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,460 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,461 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,461 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,461 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,467 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,467 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,468 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,471 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:33,736 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:33,736 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:34,737 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:34,737 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:35,737 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:35,737 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:36,738 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:36,738 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T05:37:37,593 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-11-22T05:37:37,593 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-11-22T05:37:37,596 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-11-22T05:37:37,596 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.
2024-11-22T05:37:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59., hostname=82891a793a29,42063,1732253846222, seqNum=2]
2024-11-22T05:37:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:37,613 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c3b99209a6d4a5122f7ec5ad421b3c59 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T05:37:37,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/de1672bd1d2f44fda44e3d2a84f8a3c8 is 1080, key is row0001/info:/1732253857600/Put/seqid=0
2024-11-22T05:37:37,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741837_1013 (size=12509)
2024-11-22T05:37:37,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741837_1013 (size=12509)
2024-11-22T05:37:37,640 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/de1672bd1d2f44fda44e3d2a84f8a3c8
2024-11-22T05:37:37,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/de1672bd1d2f44fda44e3d2a84f8a3c8 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/de1672bd1d2f44fda44e3d2a84f8a3c8
2024-11-22T05:37:37,656 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/de1672bd1d2f44fda44e3d2a84f8a3c8, entries=7, sequenceid=11, filesize=12.2 K
2024-11-22T05:37:37,657 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for c3b99209a6d4a5122f7ec5ad421b3c59 in 44ms, sequenceid=11, compaction requested=false
2024-11-22T05:37:37,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c3b99209a6d4a5122f7ec5ad421b3c59:

2024-11-22T05:37:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:37,659 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c3b99209a6d4a5122f7ec5ad421b3c59 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-11-22T05:37:37,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/d9e41273be3b4d3698ada649977d8e10 is 1080, key is row0008/info:/1732253857615/Put/seqid=0
2024-11-22T05:37:37,682 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741838_1014 (size=21141)
2024-11-22T05:37:37,682 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741838_1014 (size=21141)
2024-11-22T05:37:37,683 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/d9e41273be3b4d3698ada649977d8e10
2024-11-22T05:37:37,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/d9e41273be3b4d3698ada649977d8e10 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/d9e41273be3b4d3698ada649977d8e10
2024-11-22T05:37:37,698 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/d9e41273be3b4d3698ada649977d8e10, entries=15, sequenceid=29, filesize=20.6 K
2024-11-22T05:37:37,699 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=10.51 KB/10760 for c3b99209a6d4a5122f7ec5ad421b3c59 in 40ms, sequenceid=29, compaction requested=false
2024-11-22T05:37:37,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c3b99209a6d4a5122f7ec5ad421b3c59:

2024-11-22T05:37:37,700 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=32.9 K, sizeToCheck=16.0 K
2024-11-22T05:37:37,700 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:37:37,700 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/d9e41273be3b4d3698ada649977d8e10 because midkey is the same as first or last row
2024-11-22T05:37:37,739 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:37,739 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:38,739 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:38,739 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:39,693 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c3b99209a6d4a5122f7ec5ad421b3c59 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-22T05:37:39,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/c88b7825445a47648f2a574112297d85 is 1080, key is row0023/info:/1732253857660/Put/seqid=0
2024-11-22T05:37:39,707 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741839_1015 (size=16817)
2024-11-22T05:37:39,707 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741839_1015 (size=16817)
2024-11-22T05:37:39,709 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/c88b7825445a47648f2a574112297d85
2024-11-22T05:37:39,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/c88b7825445a47648f2a574112297d85 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/c88b7825445a47648f2a574112297d85
2024-11-22T05:37:39,731 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/c88b7825445a47648f2a574112297d85, entries=11, sequenceid=43, filesize=16.4 K
2024-11-22T05:37:39,733 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=10.51 KB/10760 for c3b99209a6d4a5122f7ec5ad421b3c59 in 40ms, sequenceid=43, compaction requested=true
2024-11-22T05:37:39,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c3b99209a6d4a5122f7ec5ad421b3c59:

2024-11-22T05:37:39,733 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=49.3 K, sizeToCheck=16.0 K
2024-11-22T05:37:39,733 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:37:39,733 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/d9e41273be3b4d3698ada649977d8e10 because midkey is the same as first or last row
2024-11-22T05:37:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:39,740 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:39,740 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:39,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3b99209a6d4a5122f7ec5ad421b3c59:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T05:37:39,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:37:39,748 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T05:37:39,748 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c3b99209a6d4a5122f7ec5ad421b3c59 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-11-22T05:37:39,750 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 50467 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T05:37:39,750 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1541): c3b99209a6d4a5122f7ec5ad421b3c59/info is initiating minor compaction (all files)
2024-11-22T05:37:39,750 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3b99209a6d4a5122f7ec5ad421b3c59/info in TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.
2024-11-22T05:37:39,750 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/de1672bd1d2f44fda44e3d2a84f8a3c8, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/d9e41273be3b4d3698ada649977d8e10, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/c88b7825445a47648f2a574112297d85] into tmpdir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp, totalSize=49.3 K
2024-11-22T05:37:39,751 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting de1672bd1d2f44fda44e3d2a84f8a3c8, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732253857600
2024-11-22T05:37:39,752 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9e41273be3b4d3698ada649977d8e10, keycount=15, bloomtype=ROW, size=20.6 K, encoding=NONE, compression=NONE, seqNum=29, earliestPutTs=1732253857615
2024-11-22T05:37:39,752 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting c88b7825445a47648f2a574112297d85, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732253857660
2024-11-22T05:37:39,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/7c6b5a7d080542648f359a67a4c1f604 is 1080, key is row0034/info:/1732253859695/Put/seqid=0
2024-11-22T05:37:39,790 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=c3b99209a6d4a5122f7ec5ad421b3c59, server=82891a793a29,42063,1732253846222
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-11-22T05:37:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:45008 deadline: 1732253869789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=c3b99209a6d4a5122f7ec5ad421b3c59, server=82891a793a29,42063,1732253846222
2024-11-22T05:37:39,804 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3b99209a6d4a5122f7ec5ad421b3c59#info#compaction#59 average throughput is 16.93 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T05:37:39,805 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/30ca6c185f36431fa6069d8e81680c9d is 1080, key is row0001/info:/1732253857600/Put/seqid=0
2024-11-22T05:37:39,810 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741840_1016 (size=22222)
2024-11-22T05:37:39,812 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741840_1016 (size=22222)
2024-11-22T05:37:39,813 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/7c6b5a7d080542648f359a67a4c1f604
2024-11-22T05:37:39,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/7c6b5a7d080542648f359a67a4c1f604 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/7c6b5a7d080542648f359a67a4c1f604
2024-11-22T05:37:39,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59., hostname=82891a793a29,42063,1732253846222, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59., hostname=82891a793a29,42063,1732253846222, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=c3b99209a6d4a5122f7ec5ad421b3c59, server=82891a793a29,42063,1732253846222
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-22T05:37:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59., hostname=82891a793a29,42063,1732253846222, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=c3b99209a6d4a5122f7ec5ad421b3c59, server=82891a793a29,42063,1732253846222
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-22T05:37:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59., hostname=82891a793a29,42063,1732253846222, seqNum=2 because the exception is null or not the one we care about
2024-11-22T05:37:39,829 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/7c6b5a7d080542648f359a67a4c1f604, entries=16, sequenceid=62, filesize=21.7 K
2024-11-22T05:37:39,830 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=13.66 KB/13988 for c3b99209a6d4a5122f7ec5ad421b3c59 in 82ms, sequenceid=62, compaction requested=false
2024-11-22T05:37:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c3b99209a6d4a5122f7ec5ad421b3c59:

2024-11-22T05:37:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=71.0 K, sizeToCheck=16.0 K
2024-11-22T05:37:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:37:39,831 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/7c6b5a7d080542648f359a67a4c1f604 because midkey is the same as first or last row
2024-11-22T05:37:39,838 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741841_1017 (size=40670)
2024-11-22T05:37:39,840 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741841_1017 (size=40670)
2024-11-22T05:37:39,851 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/30ca6c185f36431fa6069d8e81680c9d as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/30ca6c185f36431fa6069d8e81680c9d
2024-11-22T05:37:39,859 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c3b99209a6d4a5122f7ec5ad421b3c59/info of c3b99209a6d4a5122f7ec5ad421b3c59 into 30ca6c185f36431fa6069d8e81680c9d(size=39.7 K), total size for store is 61.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T05:37:39,859 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3b99209a6d4a5122f7ec5ad421b3c59:

2024-11-22T05:37:39,859 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59., storeName=c3b99209a6d4a5122f7ec5ad421b3c59/info, priority=13, startTime=1732253859733; duration=0sec
2024-11-22T05:37:39,859 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=61.4 K, sizeToCheck=16.0 K
2024-11-22T05:37:39,859 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:37:39,860 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/30ca6c185f36431fa6069d8e81680c9d because midkey is the same as first or last row
2024-11-22T05:37:39,860 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=61.4 K, sizeToCheck=16.0 K
2024-11-22T05:37:39,860 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:37:39,860 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/30ca6c185f36431fa6069d8e81680c9d because midkey is the same as first or last row
2024-11-22T05:37:39,860 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=61.4 K, sizeToCheck=16.0 K
2024-11-22T05:37:39,860 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:37:39,860 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/30ca6c185f36431fa6069d8e81680c9d because midkey is the same as first or last row
2024-11-22T05:37:39,860 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:37:39,860 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3b99209a6d4a5122f7ec5ad421b3c59:info
2024-11-22T05:37:40,741 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:40,741 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:41,742 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:41,742 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:42,743 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:42,743 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:43,743 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:43,743 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:44,744 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:44,744 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:45,744 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:45,744 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:46,745 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:46,745 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:47,745 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:47,745 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:48,746 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:48,746 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:49,747 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:49,747 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:49,887 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c3b99209a6d4a5122f7ec5ad421b3c59 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-22T05:37:49,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/2789ce00f2e14e328b29a26ec2ea8b8f is 1080, key is row0050/info:/1732253859750/Put/seqid=0
2024-11-22T05:37:49,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741842_1018 (size=20064)
2024-11-22T05:37:49,946 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741842_1018 (size=20064)
2024-11-22T05:37:49,947 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/2789ce00f2e14e328b29a26ec2ea8b8f
2024-11-22T05:37:49,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/2789ce00f2e14e328b29a26ec2ea8b8f as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/2789ce00f2e14e328b29a26ec2ea8b8f
2024-11-22T05:37:49,961 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/2789ce00f2e14e328b29a26ec2ea8b8f, entries=14, sequenceid=80, filesize=19.6 K
2024-11-22T05:37:49,962 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=1.05 KB/1076 for c3b99209a6d4a5122f7ec5ad421b3c59 in 75ms, sequenceid=80, compaction requested=true
2024-11-22T05:37:49,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c3b99209a6d4a5122f7ec5ad421b3c59:

2024-11-22T05:37:49,962 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=81.0 K, sizeToCheck=16.0 K
2024-11-22T05:37:49,962 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:37:49,962 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/30ca6c185f36431fa6069d8e81680c9d because midkey is the same as first or last row
2024-11-22T05:37:49,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c3b99209a6d4a5122f7ec5ad421b3c59:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T05:37:49,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:37:49,963 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T05:37:49,964 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 82956 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T05:37:49,964 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.HStore(1541): c3b99209a6d4a5122f7ec5ad421b3c59/info is initiating minor compaction (all files)
2024-11-22T05:37:49,964 INFO  [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c3b99209a6d4a5122f7ec5ad421b3c59/info in TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.
2024-11-22T05:37:49,964 INFO  [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/30ca6c185f36431fa6069d8e81680c9d, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/7c6b5a7d080542648f359a67a4c1f604, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/2789ce00f2e14e328b29a26ec2ea8b8f] into tmpdir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp, totalSize=81.0 K
2024-11-22T05:37:49,965 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] compactions.Compactor(225): Compacting 30ca6c185f36431fa6069d8e81680c9d, keycount=33, bloomtype=ROW, size=39.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732253857600
2024-11-22T05:37:49,965 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c6b5a7d080542648f359a67a4c1f604, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732253859695
2024-11-22T05:37:49,965 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] compactions.Compactor(225): Compacting 2789ce00f2e14e328b29a26ec2ea8b8f, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732253859750
2024-11-22T05:37:49,978 INFO  [RS:0;82891a793a29:42063-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c3b99209a6d4a5122f7ec5ad421b3c59#info#compaction#61 average throughput is 32.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T05:37:49,979 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/0be85574bf274bd8b08cd1eda4b22553 is 1080, key is row0001/info:/1732253857600/Put/seqid=0
2024-11-22T05:37:49,982 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741843_1019 (size=73224)
2024-11-22T05:37:49,982 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741843_1019 (size=73224)
2024-11-22T05:37:49,987 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/0be85574bf274bd8b08cd1eda4b22553 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/0be85574bf274bd8b08cd1eda4b22553
2024-11-22T05:37:49,993 INFO  [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c3b99209a6d4a5122f7ec5ad421b3c59/info of c3b99209a6d4a5122f7ec5ad421b3c59 into 0be85574bf274bd8b08cd1eda4b22553(size=71.5 K), total size for store is 71.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T05:37:49,993 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c3b99209a6d4a5122f7ec5ad421b3c59:

2024-11-22T05:37:49,993 INFO  [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59., storeName=c3b99209a6d4a5122f7ec5ad421b3c59/info, priority=13, startTime=1732253869962; duration=0sec
2024-11-22T05:37:49,993 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=71.5 K, sizeToCheck=16.0 K
2024-11-22T05:37:49,993 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:37:49,993 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=71.5 K, sizeToCheck=16.0 K
2024-11-22T05:37:49,993 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:37:49,993 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=71.5 K, sizeToCheck=16.0 K
2024-11-22T05:37:49,993 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T05:37:49,994 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:37:49,994 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:37:49,994 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c3b99209a6d4a5122f7ec5ad421b3c59:info
2024-11-22T05:37:49,995 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44627 {}] assignment.AssignmentManager(1363): Split request from 82891a793a29,42063,1732253846222, parent={ENCODED => c3b99209a6d4a5122f7ec5ad421b3c59, NAME => 'TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-22T05:37:50,000 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44627 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=82891a793a29,42063,1732253846222
2024-11-22T05:37:50,004 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44627 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=c3b99209a6d4a5122f7ec5ad421b3c59, daughterA=d7f712471bbedcd2472caa8e146dda67, daughterB=3fee3a70d358956b05e5e725f7751041
2024-11-22T05:37:50,005 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=c3b99209a6d4a5122f7ec5ad421b3c59, daughterA=d7f712471bbedcd2472caa8e146dda67, daughterB=3fee3a70d358956b05e5e725f7751041
2024-11-22T05:37:50,005 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=c3b99209a6d4a5122f7ec5ad421b3c59, daughterA=d7f712471bbedcd2472caa8e146dda67, daughterB=3fee3a70d358956b05e5e725f7751041
2024-11-22T05:37:50,005 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=c3b99209a6d4a5122f7ec5ad421b3c59, daughterA=d7f712471bbedcd2472caa8e146dda67, daughterB=3fee3a70d358956b05e5e725f7751041
2024-11-22T05:37:50,011 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c3b99209a6d4a5122f7ec5ad421b3c59, UNASSIGN}]
2024-11-22T05:37:50,013 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c3b99209a6d4a5122f7ec5ad421b3c59, UNASSIGN
2024-11-22T05:37:50,014 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c3b99209a6d4a5122f7ec5ad421b3c59, regionState=CLOSING, regionLocation=82891a793a29,42063,1732253846222
2024-11-22T05:37:50,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c3b99209a6d4a5122f7ec5ad421b3c59, UNASSIGN because future has completed
2024-11-22T05:37:50,017 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-11-22T05:37:50,017 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure c3b99209a6d4a5122f7ec5ad421b3c59, server=82891a793a29,42063,1732253846222}]
2024-11-22T05:37:50,173 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:50,173 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-11-22T05:37:50,174 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing c3b99209a6d4a5122f7ec5ad421b3c59, disabling compactions & flushes
2024-11-22T05:37:50,174 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.
2024-11-22T05:37:50,174 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.
2024-11-22T05:37:50,174 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59. after waiting 0 ms
2024-11-22T05:37:50,174 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.
2024-11-22T05:37:50,174 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing c3b99209a6d4a5122f7ec5ad421b3c59 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-22T05:37:50,178 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/9d90add7685f4c7497e834c4b7f7d4ed is 1080, key is row0064/info:/1732253869890/Put/seqid=0
2024-11-22T05:37:50,183 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741844_1020 (size=6033)
2024-11-22T05:37:50,183 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741844_1020 (size=6033)
2024-11-22T05:37:50,183 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/9d90add7685f4c7497e834c4b7f7d4ed
2024-11-22T05:37:50,189 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/.tmp/info/9d90add7685f4c7497e834c4b7f7d4ed as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/9d90add7685f4c7497e834c4b7f7d4ed
2024-11-22T05:37:50,193 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/9d90add7685f4c7497e834c4b7f7d4ed, entries=1, sequenceid=85, filesize=5.9 K
2024-11-22T05:37:50,194 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for c3b99209a6d4a5122f7ec5ad421b3c59 in 20ms, sequenceid=85, compaction requested=false
2024-11-22T05:37:50,195 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/de1672bd1d2f44fda44e3d2a84f8a3c8, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/d9e41273be3b4d3698ada649977d8e10, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/30ca6c185f36431fa6069d8e81680c9d, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/c88b7825445a47648f2a574112297d85, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/7c6b5a7d080542648f359a67a4c1f604, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/2789ce00f2e14e328b29a26ec2ea8b8f] to archive
2024-11-22T05:37:50,196 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-22T05:37:50,197 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/de1672bd1d2f44fda44e3d2a84f8a3c8 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/de1672bd1d2f44fda44e3d2a84f8a3c8
2024-11-22T05:37:50,198 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/d9e41273be3b4d3698ada649977d8e10 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/d9e41273be3b4d3698ada649977d8e10
2024-11-22T05:37:50,199 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/30ca6c185f36431fa6069d8e81680c9d to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/30ca6c185f36431fa6069d8e81680c9d
2024-11-22T05:37:50,200 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/c88b7825445a47648f2a574112297d85 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/c88b7825445a47648f2a574112297d85
2024-11-22T05:37:50,201 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/7c6b5a7d080542648f359a67a4c1f604 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/7c6b5a7d080542648f359a67a4c1f604
2024-11-22T05:37:50,202 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/2789ce00f2e14e328b29a26ec2ea8b8f to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/2789ce00f2e14e328b29a26ec2ea8b8f
2024-11-22T05:37:50,207 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/recovered.edits/88.seqid, newMaxSeqId=88, maxSeqId=1
2024-11-22T05:37:50,208 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.
2024-11-22T05:37:50,208 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for c3b99209a6d4a5122f7ec5ad421b3c59:
Waiting for close lock at 1732253870173Running coprocessor pre-close hooks at 1732253870173Disabling compacts and flushes for region at 1732253870174 (+1 ms)Disabling writes for close at 1732253870174Obtaining lock to block concurrent updates at 1732253870174Preparing flush snapshotting stores in c3b99209a6d4a5122f7ec5ad421b3c59 at 1732253870174Finished memstore snapshotting TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1732253870174Flushing stores of TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59. at 1732253870175 (+1 ms)Flushing c3b99209a6d4a5122f7ec5ad421b3c59/info: creating writer at 1732253870175Flushing c3b99209a6d4a5122f7ec5ad421b3c59/info: appending metadata at 1732253870177 (+2 ms)Flushing c3b99209a6d4a5122f7ec5ad421b3c59/info: closing flushed file at 1732253870177Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@709e18b2: reopening flushed file at 1732253870188 (+11 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for c3b99209a6d4a5122f7ec5ad421b3c59 in 20ms, sequenceid=85, compaction requested=false at 1732253870194 (+6 ms)Writing region close event to WAL at 1732253870204 (+10 ms)Running coprocessor post-close hooks at 1732253870208 (+4 ms)Closed at 1732253870208
2024-11-22T05:37:50,210 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:50,211 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c3b99209a6d4a5122f7ec5ad421b3c59, regionState=CLOSED
2024-11-22T05:37:50,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure c3b99209a6d4a5122f7ec5ad421b3c59, server=82891a793a29,42063,1732253846222 because future has completed
2024-11-22T05:37:50,215 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-22T05:37:50,215 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure c3b99209a6d4a5122f7ec5ad421b3c59, server=82891a793a29,42063,1732253846222 in 196 msec
2024-11-22T05:37:50,217 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-22T05:37:50,217 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c3b99209a6d4a5122f7ec5ad421b3c59, UNASSIGN in 204 msec
2024-11-22T05:37:50,223 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:50,226 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 2 storefiles, region=c3b99209a6d4a5122f7ec5ad421b3c59, threads=2
2024-11-22T05:37:50,227 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/0be85574bf274bd8b08cd1eda4b22553 for region: c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:50,227 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/9d90add7685f4c7497e834c4b7f7d4ed for region: c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:50,237 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/9d90add7685f4c7497e834c4b7f7d4ed, top=true
2024-11-22T05:37:50,240 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741845_1021 (size=27)
2024-11-22T05:37:50,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741845_1021 (size=27)
2024-11-22T05:37:50,245 INFO  [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/TestLogRolling-testLogRolling=c3b99209a6d4a5122f7ec5ad421b3c59-9d90add7685f4c7497e834c4b7f7d4ed for child: 3fee3a70d358956b05e5e725f7751041, parent: c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:50,245 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/9d90add7685f4c7497e834c4b7f7d4ed for region: c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:50,251 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741846_1022 (size=27)
2024-11-22T05:37:50,251 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741846_1022 (size=27)
2024-11-22T05:37:50,252 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/0be85574bf274bd8b08cd1eda4b22553 for region: c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:37:50,253 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region c3b99209a6d4a5122f7ec5ad421b3c59 Daughter A: [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/d7f712471bbedcd2472caa8e146dda67/info/0be85574bf274bd8b08cd1eda4b22553.c3b99209a6d4a5122f7ec5ad421b3c59] storefiles, Daughter B: [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/0be85574bf274bd8b08cd1eda4b22553.c3b99209a6d4a5122f7ec5ad421b3c59, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/TestLogRolling-testLogRolling=c3b99209a6d4a5122f7ec5ad421b3c59-9d90add7685f4c7497e834c4b7f7d4ed] storefiles.
2024-11-22T05:37:50,260 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741847_1023 (size=71)
2024-11-22T05:37:50,260 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741847_1023 (size=71)
2024-11-22T05:37:50,263 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:50,273 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741848_1024 (size=71)
2024-11-22T05:37:50,274 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741848_1024 (size=71)
2024-11-22T05:37:50,275 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:50,284 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/d7f712471bbedcd2472caa8e146dda67/recovered.edits/88.seqid, newMaxSeqId=88, maxSeqId=-1
2024-11-22T05:37:50,286 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/recovered.edits/88.seqid, newMaxSeqId=88, maxSeqId=-1
2024-11-22T05:37:50,288 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732253870288"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1732253870288"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1732253870288"}]},"ts":"1732253870288"}
2024-11-22T05:37:50,289 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732253870288"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732253870288"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732253870288"}]},"ts":"1732253870288"}
2024-11-22T05:37:50,289 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732253870288"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732253870288"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732253870288"}]},"ts":"1732253870288"}
2024-11-22T05:37:50,308 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d7f712471bbedcd2472caa8e146dda67, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3fee3a70d358956b05e5e725f7751041, ASSIGN}]
2024-11-22T05:37:50,309 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3fee3a70d358956b05e5e725f7751041, ASSIGN
2024-11-22T05:37:50,309 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d7f712471bbedcd2472caa8e146dda67, ASSIGN
2024-11-22T05:37:50,310 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3fee3a70d358956b05e5e725f7751041, ASSIGN; state=SPLITTING_NEW, location=82891a793a29,42063,1732253846222; forceNewPlan=false, retain=false
2024-11-22T05:37:50,311 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d7f712471bbedcd2472caa8e146dda67, ASSIGN; state=SPLITTING_NEW, location=82891a793a29,42063,1732253846222; forceNewPlan=false, retain=false
2024-11-22T05:37:50,461 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d7f712471bbedcd2472caa8e146dda67, regionState=OPENING, regionLocation=82891a793a29,42063,1732253846222
2024-11-22T05:37:50,461 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3fee3a70d358956b05e5e725f7751041, regionState=OPENING, regionLocation=82891a793a29,42063,1732253846222
2024-11-22T05:37:50,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3fee3a70d358956b05e5e725f7751041, ASSIGN because future has completed
2024-11-22T05:37:50,467 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3fee3a70d358956b05e5e725f7751041, server=82891a793a29,42063,1732253846222}]
2024-11-22T05:37:50,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d7f712471bbedcd2472caa8e146dda67, ASSIGN because future has completed
2024-11-22T05:37:50,469 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7f712471bbedcd2472caa8e146dda67, server=82891a793a29,42063,1732253846222}]
2024-11-22T05:37:50,630 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.
2024-11-22T05:37:50,631 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => d7f712471bbedcd2472caa8e146dda67, NAME => 'TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.', STARTKEY => '', ENDKEY => 'row0062'}
2024-11-22T05:37:50,632 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling d7f712471bbedcd2472caa8e146dda67
2024-11-22T05:37:50,632 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:37:50,632 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for d7f712471bbedcd2472caa8e146dda67
2024-11-22T05:37:50,633 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for d7f712471bbedcd2472caa8e146dda67
2024-11-22T05:37:50,636 INFO  [StoreOpener-d7f712471bbedcd2472caa8e146dda67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d7f712471bbedcd2472caa8e146dda67 
2024-11-22T05:37:50,638 INFO  [StoreOpener-d7f712471bbedcd2472caa8e146dda67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7f712471bbedcd2472caa8e146dda67 columnFamilyName info
2024-11-22T05:37:50,638 DEBUG [StoreOpener-d7f712471bbedcd2472caa8e146dda67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:50,651 DEBUG [StoreOpener-d7f712471bbedcd2472caa8e146dda67-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/d7f712471bbedcd2472caa8e146dda67/info/0be85574bf274bd8b08cd1eda4b22553.c3b99209a6d4a5122f7ec5ad421b3c59->hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/0be85574bf274bd8b08cd1eda4b22553-bottom
2024-11-22T05:37:50,651 INFO  [StoreOpener-d7f712471bbedcd2472caa8e146dda67-1 {}] regionserver.HStore(327): Store=d7f712471bbedcd2472caa8e146dda67/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:37:50,652 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for d7f712471bbedcd2472caa8e146dda67
2024-11-22T05:37:50,652 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/d7f712471bbedcd2472caa8e146dda67
2024-11-22T05:37:50,654 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/d7f712471bbedcd2472caa8e146dda67
2024-11-22T05:37:50,654 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for d7f712471bbedcd2472caa8e146dda67
2024-11-22T05:37:50,654 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for d7f712471bbedcd2472caa8e146dda67
2024-11-22T05:37:50,656 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for d7f712471bbedcd2472caa8e146dda67
2024-11-22T05:37:50,657 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened d7f712471bbedcd2472caa8e146dda67; next sequenceid=89; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=817786, jitterRate=0.039869531989097595}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-22T05:37:50,657 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d7f712471bbedcd2472caa8e146dda67
2024-11-22T05:37:50,658 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for d7f712471bbedcd2472caa8e146dda67:
Running coprocessor pre-open hook at 1732253870633Writing region info on filesystem at 1732253870633Initializing all the Stores at 1732253870635 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253870635Cleaning up temporary data from old regions at 1732253870654 (+19 ms)Running coprocessor post-open hooks at 1732253870658 (+4 ms)Region opened successfully at 1732253870658
2024-11-22T05:37:50,659 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67., pid=13, masterSystemTime=1732253870621
2024-11-22T05:37:50,659 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store d7f712471bbedcd2472caa8e146dda67:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T05:37:50,659 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-22T05:37:50,659 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:37:50,660 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.
2024-11-22T05:37:50,660 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1541): d7f712471bbedcd2472caa8e146dda67/info is initiating minor compaction (all files)
2024-11-22T05:37:50,660 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7f712471bbedcd2472caa8e146dda67/info in TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.
2024-11-22T05:37:50,661 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/d7f712471bbedcd2472caa8e146dda67/info/0be85574bf274bd8b08cd1eda4b22553.c3b99209a6d4a5122f7ec5ad421b3c59->hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/0be85574bf274bd8b08cd1eda4b22553-bottom] into tmpdir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/d7f712471bbedcd2472caa8e146dda67/.tmp, totalSize=71.5 K
2024-11-22T05:37:50,661 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0be85574bf274bd8b08cd1eda4b22553.c3b99209a6d4a5122f7ec5ad421b3c59, keycount=31, bloomtype=ROW, size=71.5 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732253857600
2024-11-22T05:37:50,662 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.
2024-11-22T05:37:50,662 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.
2024-11-22T05:37:50,662 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:37:50,662 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 3fee3a70d358956b05e5e725f7751041, NAME => 'TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.', STARTKEY => 'row0062', ENDKEY => ''}
2024-11-22T05:37:50,663 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:37:50,663 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:37:50,663 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:37:50,663 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:37:50,663 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d7f712471bbedcd2472caa8e146dda67, regionState=OPEN, openSeqNum=89, regionLocation=82891a793a29,42063,1732253846222
2024-11-22T05:37:50,664 INFO  [StoreOpener-3fee3a70d358956b05e5e725f7751041-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 3fee3a70d358956b05e5e725f7751041 
2024-11-22T05:37:50,665 INFO  [StoreOpener-3fee3a70d358956b05e5e725f7751041-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3fee3a70d358956b05e5e725f7751041 columnFamilyName info
2024-11-22T05:37:50,666 DEBUG [StoreOpener-3fee3a70d358956b05e5e725f7751041-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:37:50,666 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-11-22T05:37:50,666 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-11-22T05:37:50,666 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB
2024-11-22T05:37:50,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7f712471bbedcd2472caa8e146dda67, server=82891a793a29,42063,1732253846222 because future has completed
2024-11-22T05:37:50,672 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10
2024-11-22T05:37:50,672 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure d7f712471bbedcd2472caa8e146dda67, server=82891a793a29,42063,1732253846222 in 200 msec
2024-11-22T05:37:50,674 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d7f712471bbedcd2472caa8e146dda67, ASSIGN in 364 msec
2024-11-22T05:37:50,676 DEBUG [StoreOpener-3fee3a70d358956b05e5e725f7751041-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/0be85574bf274bd8b08cd1eda4b22553.c3b99209a6d4a5122f7ec5ad421b3c59->hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/0be85574bf274bd8b08cd1eda4b22553-top
2024-11-22T05:37:50,681 DEBUG [StoreOpener-3fee3a70d358956b05e5e725f7751041-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/TestLogRolling-testLogRolling=c3b99209a6d4a5122f7ec5ad421b3c59-9d90add7685f4c7497e834c4b7f7d4ed
2024-11-22T05:37:50,681 INFO  [StoreOpener-3fee3a70d358956b05e5e725f7751041-1 {}] regionserver.HStore(327): Store=3fee3a70d358956b05e5e725f7751041/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:37:50,681 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:37:50,682 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041
2024-11-22T05:37:50,683 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041
2024-11-22T05:37:50,683 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:37:50,683 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:37:50,683 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7f712471bbedcd2472caa8e146dda67#info#compaction#63 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T05:37:50,684 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/d7f712471bbedcd2472caa8e146dda67/.tmp/info/d725d1478c294df3b974b9bda4042128 is 1080, key is row0001/info:/1732253857600/Put/seqid=0
2024-11-22T05:37:50,685 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:37:50,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/.tmp/info/4d54dc897d86498fa4c7d9adc929f75e is 193, key is TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041./info:regioninfo/1732253870461/Put/seqid=0
2024-11-22T05:37:50,686 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 3fee3a70d358956b05e5e725f7751041; next sequenceid=89; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=861740, jitterRate=0.09575912356376648}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-22T05:37:50,686 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:37:50,686 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 3fee3a70d358956b05e5e725f7751041:
Running coprocessor pre-open hook at 1732253870663Writing region info on filesystem at 1732253870663Initializing all the Stores at 1732253870664 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253870664Cleaning up temporary data from old regions at 1732253870683 (+19 ms)Running coprocessor post-open hooks at 1732253870686 (+3 ms)Region opened successfully at 1732253870686
2024-11-22T05:37:50,687 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041., pid=12, masterSystemTime=1732253870621
2024-11-22T05:37:50,687 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store 3fee3a70d358956b05e5e725f7751041:info, priority=-2147483648, current under compaction store size is 2
2024-11-22T05:37:50,687 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:37:50,687 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking
2024-11-22T05:37:50,688 INFO  [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:37:50,688 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.HStore(1541): 3fee3a70d358956b05e5e725f7751041/info is initiating minor compaction (all files)
2024-11-22T05:37:50,688 INFO  [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fee3a70d358956b05e5e725f7751041/info in TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:37:50,688 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741849_1025 (size=70862)
2024-11-22T05:37:50,688 INFO  [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/0be85574bf274bd8b08cd1eda4b22553.c3b99209a6d4a5122f7ec5ad421b3c59->hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/0be85574bf274bd8b08cd1eda4b22553-top, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/TestLogRolling-testLogRolling=c3b99209a6d4a5122f7ec5ad421b3c59-9d90add7685f4c7497e834c4b7f7d4ed] into tmpdir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp, totalSize=77.4 K
2024-11-22T05:37:50,689 DEBUG [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:37:50,690 INFO  [RS_OPEN_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:37:50,690 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3fee3a70d358956b05e5e725f7751041, regionState=OPEN, openSeqNum=89, regionLocation=82891a793a29,42063,1732253846222
2024-11-22T05:37:50,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3fee3a70d358956b05e5e725f7751041, server=82891a793a29,42063,1732253846222 because future has completed
2024-11-22T05:37:50,694 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] compactions.Compactor(225): Compacting 0be85574bf274bd8b08cd1eda4b22553.c3b99209a6d4a5122f7ec5ad421b3c59, keycount=31, bloomtype=ROW, size=71.5 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732253857600
2024-11-22T05:37:50,694 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741849_1025 (size=70862)
2024-11-22T05:37:50,694 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=c3b99209a6d4a5122f7ec5ad421b3c59-9d90add7685f4c7497e834c4b7f7d4ed, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732253869890
2024-11-22T05:37:50,701 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741850_1026 (size=9847)
2024-11-22T05:37:50,701 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741850_1026 (size=9847)
2024-11-22T05:37:50,701 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/.tmp/info/4d54dc897d86498fa4c7d9adc929f75e
2024-11-22T05:37:50,703 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-22T05:37:50,703 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 3fee3a70d358956b05e5e725f7751041, server=82891a793a29,42063,1732253846222 in 235 msec
2024-11-22T05:37:50,703 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/d7f712471bbedcd2472caa8e146dda67/.tmp/info/d725d1478c294df3b974b9bda4042128 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/d7f712471bbedcd2472caa8e146dda67/info/d725d1478c294df3b974b9bda4042128
2024-11-22T05:37:50,706 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7
2024-11-22T05:37:50,706 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3fee3a70d358956b05e5e725f7751041, ASSIGN in 395 msec
2024-11-22T05:37:50,708 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=c3b99209a6d4a5122f7ec5ad421b3c59, daughterA=d7f712471bbedcd2472caa8e146dda67, daughterB=3fee3a70d358956b05e5e725f7751041 in 706 msec
2024-11-22T05:37:50,711 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in d7f712471bbedcd2472caa8e146dda67/info of d7f712471bbedcd2472caa8e146dda67 into d725d1478c294df3b974b9bda4042128(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T05:37:50,712 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7f712471bbedcd2472caa8e146dda67:

2024-11-22T05:37:50,712 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67., storeName=d7f712471bbedcd2472caa8e146dda67/info, priority=15, startTime=1732253870659; duration=0sec
2024-11-22T05:37:50,712 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:37:50,712 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7f712471bbedcd2472caa8e146dda67:info
2024-11-22T05:37:50,715 INFO  [RS:0;82891a793a29:42063-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fee3a70d358956b05e5e725f7751041#info#compaction#65 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T05:37:50,716 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/8fb5c1122d9742efb84eca604960e857 is 1080, key is row0062/info:/1732253859786/Put/seqid=0
2024-11-22T05:37:50,720 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741851_1027 (size=8359)
2024-11-22T05:37:50,720 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741851_1027 (size=8359)
2024-11-22T05:37:50,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/.tmp/ns/d1a22d27e1034830b3198d7dca9c4b11 is 43, key is default/ns:d/1732253847496/Put/seqid=0
2024-11-22T05:37:50,724 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741852_1028 (size=5153)
2024-11-22T05:37:50,724 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741852_1028 (size=5153)
2024-11-22T05:37:50,724 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/.tmp/ns/d1a22d27e1034830b3198d7dca9c4b11
2024-11-22T05:37:50,725 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/8fb5c1122d9742efb84eca604960e857 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/8fb5c1122d9742efb84eca604960e857
2024-11-22T05:37:50,731 INFO  [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 2 (all) file(s) in 3fee3a70d358956b05e5e725f7751041/info of 3fee3a70d358956b05e5e725f7751041 into 8fb5c1122d9742efb84eca604960e857(size=8.2 K), total size for store is 8.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T05:37:50,731 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:37:50,731 INFO  [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041., storeName=3fee3a70d358956b05e5e725f7751041/info, priority=14, startTime=1732253870687; duration=0sec
2024-11-22T05:37:50,731 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:37:50,731 DEBUG [RS:0;82891a793a29:42063-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fee3a70d358956b05e5e725f7751041:info
2024-11-22T05:37:50,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/.tmp/table/4949e724b7a545d2a9ee5c580d75e006 is 65, key is TestLogRolling-testLogRolling/table:state/1732253847942/Put/seqid=0
2024-11-22T05:37:50,745 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741853_1029 (size=5340)
2024-11-22T05:37:50,745 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741853_1029 (size=5340)
2024-11-22T05:37:50,746 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/.tmp/table/4949e724b7a545d2a9ee5c580d75e006
2024-11-22T05:37:50,748 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:50,748 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:50,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/.tmp/info/4d54dc897d86498fa4c7d9adc929f75e as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/info/4d54dc897d86498fa4c7d9adc929f75e
2024-11-22T05:37:50,755 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/info/4d54dc897d86498fa4c7d9adc929f75e, entries=30, sequenceid=17, filesize=9.6 K
2024-11-22T05:37:50,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/.tmp/ns/d1a22d27e1034830b3198d7dca9c4b11 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/ns/d1a22d27e1034830b3198d7dca9c4b11
2024-11-22T05:37:50,762 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/ns/d1a22d27e1034830b3198d7dca9c4b11, entries=2, sequenceid=17, filesize=5.0 K
2024-11-22T05:37:50,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/.tmp/table/4949e724b7a545d2a9ee5c580d75e006 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/table/4949e724b7a545d2a9ee5c580d75e006
2024-11-22T05:37:50,768 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/table/4949e724b7a545d2a9ee5c580d75e006, entries=2, sequenceid=17, filesize=5.2 K
2024-11-22T05:37:50,769 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 103ms, sequenceid=17, compaction requested=false
2024-11-22T05:37:50,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-22T05:37:51,749 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:51,749 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:45008 deadline: 1732253881945, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59. is not online on 82891a793a29,42063,1732253846222
2024-11-22T05:37:51,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59., hostname=82891a793a29,42063,1732253846222, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59., hostname=82891a793a29,42063,1732253846222, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59. is not online on 82891a793a29,42063,1732253846222
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-22T05:37:51,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59., hostname=82891a793a29,42063,1732253846222, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59. is not online on 82891a793a29,42063,1732253846222
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-22T05:37:51,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1732253847560.c3b99209a6d4a5122f7ec5ad421b3c59., hostname=82891a793a29,42063,1732253846222, seqNum=2 from cache
2024-11-22T05:37:52,750 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:52,750 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:53,751 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:53,751 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:54,752 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:54,752 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:55,209 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,210 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,210 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,211 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,211 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,212 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,213 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,213 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,238 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,239 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,239 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,239 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,239 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,239 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,242 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,243 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,243 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,246 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,753 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T05:37:55,753 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:55,753 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:55,755 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,755 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,756 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,757 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,757 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,758 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,759 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,760 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,785 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,786 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,786 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,786 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,786 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,787 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,790 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,790 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,790 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:55,793 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:37:56,028 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-22T05:37:56,754 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:56,754 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:57,755 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:57,755 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:58,756 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:58,756 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:59,758 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:37:59,758 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:00,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:00,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:01,760 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:01,760 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0065', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041., hostname=82891a793a29,42063,1732253846222, seqNum=89]
2024-11-22T05:38:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:02,062 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T05:38:02,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/e03145e5704b4f6183af391ae764dbff is 1080, key is row0065/info:/1732253882052/Put/seqid=0
2024-11-22T05:38:02,071 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741854_1030 (size=12509)
2024-11-22T05:38:02,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741854_1030 (size=12509)
2024-11-22T05:38:02,072 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/e03145e5704b4f6183af391ae764dbff
2024-11-22T05:38:02,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/e03145e5704b4f6183af391ae764dbff as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/e03145e5704b4f6183af391ae764dbff
2024-11-22T05:38:02,085 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/e03145e5704b4f6183af391ae764dbff, entries=7, sequenceid=99, filesize=12.2 K
2024-11-22T05:38:02,086 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for 3fee3a70d358956b05e5e725f7751041 in 24ms, sequenceid=99, compaction requested=false
2024-11-22T05:38:02,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:02,087 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-22T05:38:02,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/b0af228d97014275b738a7bfc2c92789 is 1080, key is row0072/info:/1732253882063/Put/seqid=0
2024-11-22T05:38:02,094 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741855_1031 (size=18987)
2024-11-22T05:38:02,094 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741855_1031 (size=18987)
2024-11-22T05:38:02,095 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/b0af228d97014275b738a7bfc2c92789
2024-11-22T05:38:02,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/b0af228d97014275b738a7bfc2c92789 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/b0af228d97014275b738a7bfc2c92789
2024-11-22T05:38:02,106 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/b0af228d97014275b738a7bfc2c92789, entries=13, sequenceid=115, filesize=18.5 K
2024-11-22T05:38:02,107 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=11.56 KB/11836 for 3fee3a70d358956b05e5e725f7751041 in 20ms, sequenceid=115, compaction requested=true
2024-11-22T05:38:02,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:02,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fee3a70d358956b05e5e725f7751041:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T05:38:02,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:38:02,107 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T05:38:02,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:02,108 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-22T05:38:02,109 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 39855 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T05:38:02,109 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1541): 3fee3a70d358956b05e5e725f7751041/info is initiating minor compaction (all files)
2024-11-22T05:38:02,109 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fee3a70d358956b05e5e725f7751041/info in TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:38:02,109 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/8fb5c1122d9742efb84eca604960e857, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/e03145e5704b4f6183af391ae764dbff, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/b0af228d97014275b738a7bfc2c92789] into tmpdir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp, totalSize=38.9 K
2024-11-22T05:38:02,109 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fb5c1122d9742efb84eca604960e857, keycount=3, bloomtype=ROW, size=8.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732253859786
2024-11-22T05:38:02,110 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting e03145e5704b4f6183af391ae764dbff, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732253882052
2024-11-22T05:38:02,110 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0af228d97014275b738a7bfc2c92789, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732253882063
2024-11-22T05:38:02,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/bad3543619f0491e878c7e299091073c is 1080, key is row0085/info:/1732253882088/Put/seqid=0
2024-11-22T05:38:02,117 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741856_1032 (size=17895)
2024-11-22T05:38:02,117 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741856_1032 (size=17895)
2024-11-22T05:38:02,118 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/bad3543619f0491e878c7e299091073c
2024-11-22T05:38:02,122 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fee3a70d358956b05e5e725f7751041#info#compaction#71 average throughput is 11.80 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T05:38:02,122 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/6547a6647a6c4823b65c0433f5c77410 is 1080, key is row0062/info:/1732253859786/Put/seqid=0
2024-11-22T05:38:02,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/bad3543619f0491e878c7e299091073c as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/bad3543619f0491e878c7e299091073c
2024-11-22T05:38:02,127 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741857_1033 (size=30029)
2024-11-22T05:38:02,127 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741857_1033 (size=30029)
2024-11-22T05:38:02,129 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/bad3543619f0491e878c7e299091073c, entries=12, sequenceid=130, filesize=17.5 K
2024-11-22T05:38:02,130 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=0 B/0 for 3fee3a70d358956b05e5e725f7751041 in 22ms, sequenceid=130, compaction requested=false
2024-11-22T05:38:02,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:02,132 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/6547a6647a6c4823b65c0433f5c77410 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6547a6647a6c4823b65c0433f5c77410
2024-11-22T05:38:02,137 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fee3a70d358956b05e5e725f7751041/info of 3fee3a70d358956b05e5e725f7751041 into 6547a6647a6c4823b65c0433f5c77410(size=29.3 K), total size for store is 46.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T05:38:02,137 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:02,137 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041., storeName=3fee3a70d358956b05e5e725f7751041/info, priority=13, startTime=1732253882107; duration=0sec
2024-11-22T05:38:02,137 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:38:02,137 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fee3a70d358956b05e5e725f7751041:info
2024-11-22T05:38:02,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:02,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:03,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:03,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:04,131 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T05:38:04,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/a0747b2f2c85495089d9258dc514adc8 is 1080, key is row0097/info:/1732253884112/Put/seqid=0
2024-11-22T05:38:04,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741858_1034 (size=12516)
2024-11-22T05:38:04,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741858_1034 (size=12516)
2024-11-22T05:38:04,150 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/a0747b2f2c85495089d9258dc514adc8
2024-11-22T05:38:04,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/a0747b2f2c85495089d9258dc514adc8 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/a0747b2f2c85495089d9258dc514adc8
2024-11-22T05:38:04,160 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/a0747b2f2c85495089d9258dc514adc8, entries=7, sequenceid=141, filesize=12.2 K
2024-11-22T05:38:04,161 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=13.66 KB/13988 for 3fee3a70d358956b05e5e725f7751041 in 30ms, sequenceid=141, compaction requested=true
2024-11-22T05:38:04,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:04,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fee3a70d358956b05e5e725f7751041:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T05:38:04,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:38:04,161 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T05:38:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:04,162 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-22T05:38:04,162 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 60440 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T05:38:04,162 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1541): 3fee3a70d358956b05e5e725f7751041/info is initiating minor compaction (all files)
2024-11-22T05:38:04,162 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fee3a70d358956b05e5e725f7751041/info in TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:38:04,163 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6547a6647a6c4823b65c0433f5c77410, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/bad3543619f0491e878c7e299091073c, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/a0747b2f2c85495089d9258dc514adc8] into tmpdir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp, totalSize=59.0 K
2024-11-22T05:38:04,163 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6547a6647a6c4823b65c0433f5c77410, keycount=23, bloomtype=ROW, size=29.3 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732253859786
2024-11-22T05:38:04,163 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting bad3543619f0491e878c7e299091073c, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732253882088
2024-11-22T05:38:04,164 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0747b2f2c85495089d9258dc514adc8, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732253884112
2024-11-22T05:38:04,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/19e142b9099c4544a2bf589ae1fda93d is 1080, key is row0104/info:/1732253884132/Put/seqid=0
2024-11-22T05:38:04,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741859_1035 (size=20078)
2024-11-22T05:38:04,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741859_1035 (size=20078)
2024-11-22T05:38:04,172 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/19e142b9099c4544a2bf589ae1fda93d
2024-11-22T05:38:04,175 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fee3a70d358956b05e5e725f7751041#info#compaction#74 average throughput is 43.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T05:38:04,175 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/2c87b271802f4abf9128b71c08d6665e is 1080, key is row0062/info:/1732253859786/Put/seqid=0
2024-11-22T05:38:04,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/19e142b9099c4544a2bf589ae1fda93d as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/19e142b9099c4544a2bf589ae1fda93d
2024-11-22T05:38:04,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741860_1036 (size=50638)
2024-11-22T05:38:04,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741860_1036 (size=50638)
2024-11-22T05:38:04,185 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/19e142b9099c4544a2bf589ae1fda93d, entries=14, sequenceid=158, filesize=19.6 K
2024-11-22T05:38:04,186 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=9.46 KB/9684 for 3fee3a70d358956b05e5e725f7751041 in 24ms, sequenceid=158, compaction requested=false
2024-11-22T05:38:04,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:04,188 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-22T05:38:04,189 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/2c87b271802f4abf9128b71c08d6665e as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/2c87b271802f4abf9128b71c08d6665e
2024-11-22T05:38:04,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/d70e16e396d84399844f407b4b68010f is 1080, key is row0118/info:/1732253884163/Put/seqid=0
2024-11-22T05:38:04,195 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fee3a70d358956b05e5e725f7751041/info of 3fee3a70d358956b05e5e725f7751041 into 2c87b271802f4abf9128b71c08d6665e(size=49.5 K), total size for store is 69.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T05:38:04,195 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:04,195 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041., storeName=3fee3a70d358956b05e5e725f7751041/info, priority=13, startTime=1732253884161; duration=0sec
2024-11-22T05:38:04,195 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:38:04,195 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fee3a70d358956b05e5e725f7751041:info
2024-11-22T05:38:04,197 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741861_1037 (size=15750)
2024-11-22T05:38:04,197 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741861_1037 (size=15750)
2024-11-22T05:38:04,198 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/d70e16e396d84399844f407b4b68010f
2024-11-22T05:38:04,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/d70e16e396d84399844f407b4b68010f as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/d70e16e396d84399844f407b4b68010f
2024-11-22T05:38:04,208 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/d70e16e396d84399844f407b4b68010f, entries=10, sequenceid=171, filesize=15.4 K
2024-11-22T05:38:04,209 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=1.05 KB/1076 for 3fee3a70d358956b05e5e725f7751041 in 21ms, sequenceid=171, compaction requested=true
2024-11-22T05:38:04,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:04,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fee3a70d358956b05e5e725f7751041:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T05:38:04,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:38:04,210 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T05:38:04,211 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 86466 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T05:38:04,211 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1541): 3fee3a70d358956b05e5e725f7751041/info is initiating minor compaction (all files)
2024-11-22T05:38:04,211 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fee3a70d358956b05e5e725f7751041/info in TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:38:04,211 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/2c87b271802f4abf9128b71c08d6665e, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/19e142b9099c4544a2bf589ae1fda93d, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/d70e16e396d84399844f407b4b68010f] into tmpdir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp, totalSize=84.4 K
2024-11-22T05:38:04,211 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c87b271802f4abf9128b71c08d6665e, keycount=42, bloomtype=ROW, size=49.5 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732253859786
2024-11-22T05:38:04,212 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19e142b9099c4544a2bf589ae1fda93d, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732253884132
2024-11-22T05:38:04,212 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting d70e16e396d84399844f407b4b68010f, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732253884163
2024-11-22T05:38:04,222 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fee3a70d358956b05e5e725f7751041#info#compaction#76 average throughput is 67.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T05:38:04,223 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/137b89a25d7542eabb5863d900476ffd is 1080, key is row0062/info:/1732253859786/Put/seqid=0
2024-11-22T05:38:04,238 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741862_1038 (size=76753)
2024-11-22T05:38:04,238 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741862_1038 (size=76753)
2024-11-22T05:38:04,244 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/137b89a25d7542eabb5863d900476ffd as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/137b89a25d7542eabb5863d900476ffd
2024-11-22T05:38:04,250 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fee3a70d358956b05e5e725f7751041/info of 3fee3a70d358956b05e5e725f7751041 into 137b89a25d7542eabb5863d900476ffd(size=75.0 K), total size for store is 75.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T05:38:04,250 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:04,250 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041., storeName=3fee3a70d358956b05e5e725f7751041/info, priority=13, startTime=1732253884209; duration=0sec
2024-11-22T05:38:04,250 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:38:04,250 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fee3a70d358956b05e5e725f7751041:info
2024-11-22T05:38:04,764 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:04,764 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:05,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:05,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:06,256 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T05:38:06,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/a591ac07fb24462380802621ab3837b3 is 1080, key is row0128/info:/1732253884190/Put/seqid=0
2024-11-22T05:38:06,267 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741863_1039 (size=12516)
2024-11-22T05:38:06,267 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741863_1039 (size=12516)
2024-11-22T05:38:06,267 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/a591ac07fb24462380802621ab3837b3
2024-11-22T05:38:06,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/a591ac07fb24462380802621ab3837b3 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/a591ac07fb24462380802621ab3837b3
2024-11-22T05:38:06,280 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/a591ac07fb24462380802621ab3837b3, entries=7, sequenceid=183, filesize=12.2 K
2024-11-22T05:38:06,280 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for 3fee3a70d358956b05e5e725f7751041 in 24ms, sequenceid=183, compaction requested=false
2024-11-22T05:38:06,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:06,282 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-22T05:38:06,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/6900c18cea4843918b21caa4ed5d57ac is 1080, key is row0135/info:/1732253886258/Put/seqid=0
2024-11-22T05:38:06,293 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741864_1040 (size=17906)
2024-11-22T05:38:06,293 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741864_1040 (size=17906)
2024-11-22T05:38:06,294 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/6900c18cea4843918b21caa4ed5d57ac
2024-11-22T05:38:06,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/6900c18cea4843918b21caa4ed5d57ac as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6900c18cea4843918b21caa4ed5d57ac
2024-11-22T05:38:06,305 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6900c18cea4843918b21caa4ed5d57ac, entries=12, sequenceid=198, filesize=17.5 K
2024-11-22T05:38:06,305 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=11.56 KB/11836 for 3fee3a70d358956b05e5e725f7751041 in 23ms, sequenceid=198, compaction requested=true
2024-11-22T05:38:06,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:06,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fee3a70d358956b05e5e725f7751041:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T05:38:06,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:38:06,306 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T05:38:06,307 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 107175 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T05:38:06,307 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1541): 3fee3a70d358956b05e5e725f7751041/info is initiating minor compaction (all files)
2024-11-22T05:38:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:06,307 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fee3a70d358956b05e5e725f7751041/info in TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:38:06,307 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/137b89a25d7542eabb5863d900476ffd, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/a591ac07fb24462380802621ab3837b3, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6900c18cea4843918b21caa4ed5d57ac] into tmpdir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp, totalSize=104.7 K
2024-11-22T05:38:06,307 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-22T05:38:06,308 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting 137b89a25d7542eabb5863d900476ffd, keycount=66, bloomtype=ROW, size=75.0 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732253859786
2024-11-22T05:38:06,308 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting a591ac07fb24462380802621ab3837b3, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732253884190
2024-11-22T05:38:06,308 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6900c18cea4843918b21caa4ed5d57ac, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732253886258
2024-11-22T05:38:06,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/6a6ed6181c3a451dbd8f1765d1885a91 is 1080, key is row0147/info:/1732253886283/Put/seqid=0
2024-11-22T05:38:06,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741865_1041 (size=19000)
2024-11-22T05:38:06,316 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741865_1041 (size=19000)
2024-11-22T05:38:06,316 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/6a6ed6181c3a451dbd8f1765d1885a91
2024-11-22T05:38:06,321 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fee3a70d358956b05e5e725f7751041#info#compaction#80 average throughput is 29.07 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T05:38:06,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/6a6ed6181c3a451dbd8f1765d1885a91 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6a6ed6181c3a451dbd8f1765d1885a91
2024-11-22T05:38:06,322 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/9c65c2c378004c2996c7618bf5c5f84a is 1080, key is row0062/info:/1732253859786/Put/seqid=0
2024-11-22T05:38:06,325 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741866_1042 (size=97341)
2024-11-22T05:38:06,325 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741866_1042 (size=97341)
2024-11-22T05:38:06,326 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6a6ed6181c3a451dbd8f1765d1885a91, entries=13, sequenceid=214, filesize=18.6 K
2024-11-22T05:38:06,327 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=1.05 KB/1076 for 3fee3a70d358956b05e5e725f7751041 in 20ms, sequenceid=214, compaction requested=false
2024-11-22T05:38:06,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:06,330 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/9c65c2c378004c2996c7618bf5c5f84a as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/9c65c2c378004c2996c7618bf5c5f84a
2024-11-22T05:38:06,334 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fee3a70d358956b05e5e725f7751041/info of 3fee3a70d358956b05e5e725f7751041 into 9c65c2c378004c2996c7618bf5c5f84a(size=95.1 K), total size for store is 113.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T05:38:06,335 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:06,335 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041., storeName=3fee3a70d358956b05e5e725f7751041/info, priority=13, startTime=1732253886306; duration=0sec
2024-11-22T05:38:06,335 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:38:06,335 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fee3a70d358956b05e5e725f7751041:info
2024-11-22T05:38:06,766 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:06,766 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:07,533 INFO  [master/82891a793a29:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-22T05:38:07,533 INFO  [master/82891a793a29:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-22T05:38:07,767 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:07,767 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:08,327 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T05:38:08,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/3d339ff028b6452ab06318443c6b794c is 1080, key is row0160/info:/1732253886308/Put/seqid=0
2024-11-22T05:38:08,341 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741867_1043 (size=12516)
2024-11-22T05:38:08,341 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741867_1043 (size=12516)
2024-11-22T05:38:08,342 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/3d339ff028b6452ab06318443c6b794c
2024-11-22T05:38:08,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/3d339ff028b6452ab06318443c6b794c as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/3d339ff028b6452ab06318443c6b794c
2024-11-22T05:38:08,353 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/3d339ff028b6452ab06318443c6b794c, entries=7, sequenceid=225, filesize=12.2 K
2024-11-22T05:38:08,354 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for 3fee3a70d358956b05e5e725f7751041 in 27ms, sequenceid=225, compaction requested=true
2024-11-22T05:38:08,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:08,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fee3a70d358956b05e5e725f7751041:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T05:38:08,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:38:08,354 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T05:38:08,355 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 128857 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T05:38:08,355 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1541): 3fee3a70d358956b05e5e725f7751041/info is initiating minor compaction (all files)
2024-11-22T05:38:08,355 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fee3a70d358956b05e5e725f7751041/info in TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:38:08,355 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/9c65c2c378004c2996c7618bf5c5f84a, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6a6ed6181c3a451dbd8f1765d1885a91, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/3d339ff028b6452ab06318443c6b794c] into tmpdir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp, totalSize=125.8 K
2024-11-22T05:38:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:08,356 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c65c2c378004c2996c7618bf5c5f84a, keycount=85, bloomtype=ROW, size=95.1 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732253859786
2024-11-22T05:38:08,356 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-11-22T05:38:08,356 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a6ed6181c3a451dbd8f1765d1885a91, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732253886283
2024-11-22T05:38:08,356 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d339ff028b6452ab06318443c6b794c, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732253886308
2024-11-22T05:38:08,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/7aeaf42787984d59b5bb592ef0950cf8 is 1080, key is row0167/info:/1732253888328/Put/seqid=0
2024-11-22T05:38:08,363 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741868_1044 (size=21156)
2024-11-22T05:38:08,363 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741868_1044 (size=21156)
2024-11-22T05:38:08,364 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/7aeaf42787984d59b5bb592ef0950cf8
2024-11-22T05:38:08,369 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fee3a70d358956b05e5e725f7751041#info#compaction#83 average throughput is 35.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T05:38:08,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/7aeaf42787984d59b5bb592ef0950cf8 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/7aeaf42787984d59b5bb592ef0950cf8
2024-11-22T05:38:08,370 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/61006042e69e4f8d8e941434199f56ac is 1080, key is row0062/info:/1732253859786/Put/seqid=0
2024-11-22T05:38:08,373 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741869_1045 (size=119007)
2024-11-22T05:38:08,373 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741869_1045 (size=119007)
2024-11-22T05:38:08,376 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/7aeaf42787984d59b5bb592ef0950cf8, entries=15, sequenceid=243, filesize=20.7 K
2024-11-22T05:38:08,377 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=9.46 KB/9684 for 3fee3a70d358956b05e5e725f7751041 in 21ms, sequenceid=243, compaction requested=false
2024-11-22T05:38:08,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:08,379 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-22T05:38:08,380 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/61006042e69e4f8d8e941434199f56ac as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/61006042e69e4f8d8e941434199f56ac
2024-11-22T05:38:08,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/1ca878c1835b45fc858ab5a5c7a3c74b is 1080, key is row0182/info:/1732253888357/Put/seqid=0
2024-11-22T05:38:08,386 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fee3a70d358956b05e5e725f7751041/info of 3fee3a70d358956b05e5e725f7751041 into 61006042e69e4f8d8e941434199f56ac(size=116.2 K), total size for store is 136.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T05:38:08,386 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:08,386 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041., storeName=3fee3a70d358956b05e5e725f7751041/info, priority=13, startTime=1732253888354; duration=0sec
2024-11-22T05:38:08,386 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:38:08,386 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fee3a70d358956b05e5e725f7751041:info
2024-11-22T05:38:08,387 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741870_1046 (size=16829)
2024-11-22T05:38:08,387 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741870_1046 (size=16829)
2024-11-22T05:38:08,388 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/1ca878c1835b45fc858ab5a5c7a3c74b
2024-11-22T05:38:08,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/1ca878c1835b45fc858ab5a5c7a3c74b as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/1ca878c1835b45fc858ab5a5c7a3c74b
2024-11-22T05:38:08,397 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/1ca878c1835b45fc858ab5a5c7a3c74b, entries=11, sequenceid=257, filesize=16.4 K
2024-11-22T05:38:08,398 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=0 B/0 for 3fee3a70d358956b05e5e725f7751041 in 19ms, sequenceid=257, compaction requested=true
2024-11-22T05:38:08,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:08,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fee3a70d358956b05e5e725f7751041:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T05:38:08,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:38:08,398 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T05:38:08,399 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 156992 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T05:38:08,399 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1541): 3fee3a70d358956b05e5e725f7751041/info is initiating minor compaction (all files)
2024-11-22T05:38:08,399 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fee3a70d358956b05e5e725f7751041/info in TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:38:08,399 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/61006042e69e4f8d8e941434199f56ac, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/7aeaf42787984d59b5bb592ef0950cf8, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/1ca878c1835b45fc858ab5a5c7a3c74b] into tmpdir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp, totalSize=153.3 K
2024-11-22T05:38:08,399 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61006042e69e4f8d8e941434199f56ac, keycount=105, bloomtype=ROW, size=116.2 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732253859786
2024-11-22T05:38:08,400 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aeaf42787984d59b5bb592ef0950cf8, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732253888328
2024-11-22T05:38:08,400 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ca878c1835b45fc858ab5a5c7a3c74b, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732253888357
2024-11-22T05:38:08,410 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fee3a70d358956b05e5e725f7751041#info#compaction#85 average throughput is 67.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T05:38:08,411 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/c835bae53704450aaf0510c62d479deb is 1080, key is row0062/info:/1732253859786/Put/seqid=0
2024-11-22T05:38:08,413 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741871_1047 (size=147343)
2024-11-22T05:38:08,413 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741871_1047 (size=147343)
2024-11-22T05:38:08,418 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/c835bae53704450aaf0510c62d479deb as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/c835bae53704450aaf0510c62d479deb
2024-11-22T05:38:08,423 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fee3a70d358956b05e5e725f7751041/info of 3fee3a70d358956b05e5e725f7751041 into c835bae53704450aaf0510c62d479deb(size=143.9 K), total size for store is 143.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T05:38:08,423 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:08,423 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041., storeName=3fee3a70d358956b05e5e725f7751041/info, priority=13, startTime=1732253888398; duration=0sec
2024-11-22T05:38:08,423 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:38:08,424 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fee3a70d358956b05e5e725f7751041:info
2024-11-22T05:38:08,769 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:08,769 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:09,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:09,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:10,401 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T05:38:10,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/adb91e48731d42dd8df4d3c61d73726e is 1080, key is row0193/info:/1732253890382/Put/seqid=0
2024-11-22T05:38:10,410 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741872_1048 (size=12523)
2024-11-22T05:38:10,411 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741872_1048 (size=12523)
2024-11-22T05:38:10,411 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/adb91e48731d42dd8df4d3c61d73726e
2024-11-22T05:38:10,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/adb91e48731d42dd8df4d3c61d73726e as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/adb91e48731d42dd8df4d3c61d73726e
2024-11-22T05:38:10,423 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/adb91e48731d42dd8df4d3c61d73726e, entries=7, sequenceid=269, filesize=12.2 K
2024-11-22T05:38:10,424 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for 3fee3a70d358956b05e5e725f7751041 in 23ms, sequenceid=269, compaction requested=false
2024-11-22T05:38:10,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:10,425 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-22T05:38:10,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/284b1e7c68a04a2dbc78e6cb6d52d18b is 1080, key is row0200/info:/1732253890402/Put/seqid=0
2024-11-22T05:38:10,434 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741873_1049 (size=19013)
2024-11-22T05:38:10,434 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741873_1049 (size=19013)
2024-11-22T05:38:10,435 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/284b1e7c68a04a2dbc78e6cb6d52d18b
2024-11-22T05:38:10,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/284b1e7c68a04a2dbc78e6cb6d52d18b as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/284b1e7c68a04a2dbc78e6cb6d52d18b
2024-11-22T05:38:10,444 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/284b1e7c68a04a2dbc78e6cb6d52d18b, entries=13, sequenceid=285, filesize=18.6 K
2024-11-22T05:38:10,445 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=12.61 KB/12912 for 3fee3a70d358956b05e5e725f7751041 in 20ms, sequenceid=285, compaction requested=true
2024-11-22T05:38:10,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:10,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fee3a70d358956b05e5e725f7751041:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T05:38:10,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:38:10,445 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T05:38:10,447 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 178879 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T05:38:10,447 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1541): 3fee3a70d358956b05e5e725f7751041/info is initiating minor compaction (all files)
2024-11-22T05:38:10,447 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fee3a70d358956b05e5e725f7751041/info in TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:38:10,447 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/c835bae53704450aaf0510c62d479deb, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/adb91e48731d42dd8df4d3c61d73726e, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/284b1e7c68a04a2dbc78e6cb6d52d18b] into tmpdir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp, totalSize=174.7 K
2024-11-22T05:38:10,447 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting c835bae53704450aaf0510c62d479deb, keycount=131, bloomtype=ROW, size=143.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732253859786
2024-11-22T05:38:10,448 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting adb91e48731d42dd8df4d3c61d73726e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732253890382
2024-11-22T05:38:10,448 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting 284b1e7c68a04a2dbc78e6cb6d52d18b, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732253890402
2024-11-22T05:38:10,460 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fee3a70d358956b05e5e725f7751041#info#compaction#88 average throughput is 38.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T05:38:10,460 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/9c6eb1b9271d474f8f84fbf85178f112 is 1080, key is row0062/info:/1732253859786/Put/seqid=0
2024-11-22T05:38:10,463 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741874_1050 (size=169029)
2024-11-22T05:38:10,463 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741874_1050 (size=169029)
2024-11-22T05:38:10,468 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/9c6eb1b9271d474f8f84fbf85178f112 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/9c6eb1b9271d474f8f84fbf85178f112
2024-11-22T05:38:10,474 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fee3a70d358956b05e5e725f7751041/info of 3fee3a70d358956b05e5e725f7751041 into 9c6eb1b9271d474f8f84fbf85178f112(size=165.1 K), total size for store is 165.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T05:38:10,474 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:10,474 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041., storeName=3fee3a70d358956b05e5e725f7751041/info, priority=13, startTime=1732253890445; duration=0sec
2024-11-22T05:38:10,474 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:38:10,474 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fee3a70d358956b05e5e725f7751041:info
2024-11-22T05:38:10,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:10,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:11,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:11,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:12,358 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20340
2024-11-22T05:38:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:12,452 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-22T05:38:12,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/c61866c95490462cb105609f3a17d170 is 1080, key is row0213/info:/1732253890426/Put/seqid=0
2024-11-22T05:38:12,464 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741875_1051 (size=19013)
2024-11-22T05:38:12,465 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741875_1051 (size=19013)
2024-11-22T05:38:12,465 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/c61866c95490462cb105609f3a17d170
2024-11-22T05:38:12,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/c61866c95490462cb105609f3a17d170 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/c61866c95490462cb105609f3a17d170
2024-11-22T05:38:12,477 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/c61866c95490462cb105609f3a17d170, entries=13, sequenceid=302, filesize=18.6 K
2024-11-22T05:38:12,478 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=9.46 KB/9684 for 3fee3a70d358956b05e5e725f7751041 in 27ms, sequenceid=302, compaction requested=false
2024-11-22T05:38:12,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:12,479 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-22T05:38:12,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/b7a273b98c6e43048b606aabbddb3762 is 1080, key is row0226/info:/1732253892454/Put/seqid=0
2024-11-22T05:38:12,487 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741876_1052 (size=15760)
2024-11-22T05:38:12,487 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741876_1052 (size=15760)
2024-11-22T05:38:12,488 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/b7a273b98c6e43048b606aabbddb3762
2024-11-22T05:38:12,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/b7a273b98c6e43048b606aabbddb3762 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/b7a273b98c6e43048b606aabbddb3762
2024-11-22T05:38:12,499 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/b7a273b98c6e43048b606aabbddb3762, entries=10, sequenceid=315, filesize=15.4 K
2024-11-22T05:38:12,500 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=11.56 KB/11836 for 3fee3a70d358956b05e5e725f7751041 in 21ms, sequenceid=315, compaction requested=true
2024-11-22T05:38:12,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:12,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fee3a70d358956b05e5e725f7751041:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T05:38:12,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:38:12,500 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T05:38:12,502 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 203802 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T05:38:12,502 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1541): 3fee3a70d358956b05e5e725f7751041/info is initiating minor compaction (all files)
2024-11-22T05:38:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42063 {}] regionserver.HRegion(8855): Flush requested on 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:12,502 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fee3a70d358956b05e5e725f7751041/info in TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:38:12,502 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-22T05:38:12,502 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/9c6eb1b9271d474f8f84fbf85178f112, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/c61866c95490462cb105609f3a17d170, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/b7a273b98c6e43048b606aabbddb3762] into tmpdir=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp, totalSize=199.0 K
2024-11-22T05:38:12,502 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c6eb1b9271d474f8f84fbf85178f112, keycount=151, bloomtype=ROW, size=165.1 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732253859786
2024-11-22T05:38:12,503 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting c61866c95490462cb105609f3a17d170, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732253890426
2024-11-22T05:38:12,503 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7a273b98c6e43048b606aabbddb3762, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732253892454
2024-11-22T05:38:12,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/a204f05b552a4e7690e0f48375ef5193 is 1080, key is row0236/info:/1732253892480/Put/seqid=0
2024-11-22T05:38:12,511 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741877_1053 (size=17918)
2024-11-22T05:38:12,512 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741877_1053 (size=17918)
2024-11-22T05:38:12,513 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/a204f05b552a4e7690e0f48375ef5193
2024-11-22T05:38:12,518 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fee3a70d358956b05e5e725f7751041#info#compaction#92 average throughput is 44.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T05:38:12,519 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/eb8929025d394936a22641825483ba19 is 1080, key is row0062/info:/1732253859786/Put/seqid=0
2024-11-22T05:38:12,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/a204f05b552a4e7690e0f48375ef5193 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/a204f05b552a4e7690e0f48375ef5193
2024-11-22T05:38:12,521 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741878_1054 (size=193956)
2024-11-22T05:38:12,522 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741878_1054 (size=193956)
2024-11-22T05:38:12,526 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/a204f05b552a4e7690e0f48375ef5193, entries=12, sequenceid=330, filesize=17.5 K
2024-11-22T05:38:12,528 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=9.46 KB/9684 for 3fee3a70d358956b05e5e725f7751041 in 25ms, sequenceid=330, compaction requested=false
2024-11-22T05:38:12,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:12,528 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/eb8929025d394936a22641825483ba19 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/eb8929025d394936a22641825483ba19
2024-11-22T05:38:12,534 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fee3a70d358956b05e5e725f7751041/info of 3fee3a70d358956b05e5e725f7751041 into eb8929025d394936a22641825483ba19(size=189.4 K), total size for store is 206.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T05:38:12,534 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:12,534 INFO  [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041., storeName=3fee3a70d358956b05e5e725f7751041/info, priority=13, startTime=1732253892500; duration=0sec
2024-11-22T05:38:12,534 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T05:38:12,534 DEBUG [RS:0;82891a793a29:42063-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fee3a70d358956b05e5e725f7751041:info
2024-11-22T05:38:12,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:12,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:13,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:13,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:14,526 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-11-22T05:38:14,526 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42063%2C1732253846222.1732253894526
2024-11-22T05:38:14,537 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,537 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,537 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,537 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,537 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,537 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/WALs/82891a793a29,42063,1732253846222/82891a793a29%2C42063%2C1732253846222.1732253846855 with entries=318, filesize=310.28 KB; new WAL /user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/WALs/82891a793a29,42063,1732253846222/82891a793a29%2C42063%2C1732253846222.1732253894526
2024-11-22T05:38:14,539 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37343:37343),(127.0.0.1/127.0.0.1:46427:46427)]
2024-11-22T05:38:14,539 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/WALs/82891a793a29,42063,1732253846222/82891a793a29%2C42063%2C1732253846222.1732253846855 is not closed yet, will try archiving it next time
2024-11-22T05:38:14,539 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741833_1009 (size=317731)
2024-11-22T05:38:14,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741833_1009 (size=317731)
2024-11-22T05:38:14,543 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 3fee3a70d358956b05e5e725f7751041 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-22T05:38:14,547 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/d3871d7da6e54bea8b4bea034cce194d is 1080, key is row0248/info:/1732253892503/Put/seqid=0
2024-11-22T05:38:14,551 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741880_1056 (size=14681)
2024-11-22T05:38:14,552 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741880_1056 (size=14681)
2024-11-22T05:38:14,552 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/d3871d7da6e54bea8b4bea034cce194d
2024-11-22T05:38:14,558 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/.tmp/info/d3871d7da6e54bea8b4bea034cce194d as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/d3871d7da6e54bea8b4bea034cce194d
2024-11-22T05:38:14,562 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/d3871d7da6e54bea8b4bea034cce194d, entries=9, sequenceid=343, filesize=14.3 K
2024-11-22T05:38:14,562 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=0 B/0 for 3fee3a70d358956b05e5e725f7751041 in 19ms, sequenceid=343, compaction requested=true
2024-11-22T05:38:14,563 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3fee3a70d358956b05e5e725f7751041:

2024-11-22T05:38:14,563 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB
2024-11-22T05:38:14,566 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/.tmp/info/67ab2e32b8284df3981400e6d840de35 is 193, key is TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041./info:regioninfo/1732253870690/Put/seqid=0
2024-11-22T05:38:14,570 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741881_1057 (size=6223)
2024-11-22T05:38:14,571 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741881_1057 (size=6223)
2024-11-22T05:38:14,571 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/.tmp/info/67ab2e32b8284df3981400e6d840de35
2024-11-22T05:38:14,575 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/.tmp/info/67ab2e32b8284df3981400e6d840de35 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/info/67ab2e32b8284df3981400e6d840de35
2024-11-22T05:38:14,579 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/info/67ab2e32b8284df3981400e6d840de35, entries=5, sequenceid=21, filesize=6.1 K
2024-11-22T05:38:14,580 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 17ms, sequenceid=21, compaction requested=false
2024-11-22T05:38:14,580 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-22T05:38:14,580 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d7f712471bbedcd2472caa8e146dda67:

2024-11-22T05:38:14,580 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C42063%2C1732253846222.1732253894580
2024-11-22T05:38:14,585 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,585 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,585 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,585 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,585 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,585 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/WALs/82891a793a29,42063,1732253846222/82891a793a29%2C42063%2C1732253846222.1732253894526 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/WALs/82891a793a29,42063,1732253846222/82891a793a29%2C42063%2C1732253846222.1732253894580
2024-11-22T05:38:14,586 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37343:37343),(127.0.0.1/127.0.0.1:46427:46427)]
2024-11-22T05:38:14,586 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/WALs/82891a793a29,42063,1732253846222/82891a793a29%2C42063%2C1732253846222.1732253894526 is not closed yet, will try archiving it next time
2024-11-22T05:38:14,586 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/WALs/82891a793a29,42063,1732253846222/82891a793a29%2C42063%2C1732253846222.1732253846855 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/oldWALs/82891a793a29%2C42063%2C1732253846222.1732253846855
2024-11-22T05:38:14,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741879_1055 (size=731)
2024-11-22T05:38:14,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741879_1055 (size=731)
2024-11-22T05:38:14,587 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-11-22T05:38:14,587 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/WALs/82891a793a29,42063,1732253846222/82891a793a29%2C42063%2C1732253846222.1732253894526 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/oldWALs/82891a793a29%2C42063%2C1732253846222.1732253894526
2024-11-22T05:38:14,687 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-22T05:38:14,687 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T05:38:14,687 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:38:14,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:38:14,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:38:14,687 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-22T05:38:14,688 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-22T05:38:14,688 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=901313716, stopped=false
2024-11-22T05:38:14,688 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=82891a793a29,44627,1732253846045
2024-11-22T05:38:14,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T05:38:14,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T05:38:14,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:14,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:14,759 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T05:38:14,760 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T05:38:14,760 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:38:14,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:38:14,761 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '82891a793a29,42063,1732253846222' *****
2024-11-22T05:38:14,761 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-22T05:38:14,762 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:38:14,762 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:38:14,762 INFO  [RS:0;82891a793a29:42063 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-22T05:38:14,763 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-22T05:38:14,763 INFO  [RS:0;82891a793a29:42063 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-22T05:38:14,763 INFO  [RS:0;82891a793a29:42063 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-22T05:38:14,763 INFO  [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(3091): Received CLOSE for 3fee3a70d358956b05e5e725f7751041
2024-11-22T05:38:14,764 INFO  [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(3091): Received CLOSE for d7f712471bbedcd2472caa8e146dda67
2024-11-22T05:38:14,764 INFO  [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(959): stopping server 82891a793a29,42063,1732253846222
2024-11-22T05:38:14,764 INFO  [RS:0;82891a793a29:42063 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T05:38:14,764 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3fee3a70d358956b05e5e725f7751041, disabling compactions & flushes
2024-11-22T05:38:14,764 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:38:14,764 INFO  [RS:0;82891a793a29:42063 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;82891a793a29:42063.
2024-11-22T05:38:14,764 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:38:14,764 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041. after waiting 0 ms
2024-11-22T05:38:14,764 DEBUG [RS:0;82891a793a29:42063 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:38:14,764 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:38:14,764 DEBUG [RS:0;82891a793a29:42063 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:38:14,764 INFO  [RS:0;82891a793a29:42063 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-22T05:38:14,764 INFO  [RS:0;82891a793a29:42063 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-22T05:38:14,764 INFO  [RS:0;82891a793a29:42063 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-22T05:38:14,764 INFO  [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-22T05:38:14,765 INFO  [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-11-22T05:38:14,765 DEBUG [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(1325): Online Regions={3fee3a70d358956b05e5e725f7751041=TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041., 1588230740=hbase:meta,,1.1588230740, d7f712471bbedcd2472caa8e146dda67=TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.}
2024-11-22T05:38:14,765 DEBUG [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3fee3a70d358956b05e5e725f7751041, d7f712471bbedcd2472caa8e146dda67
2024-11-22T05:38:14,765 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T05:38:14,765 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T05:38:14,765 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T05:38:14,765 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T05:38:14,765 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T05:38:14,765 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/0be85574bf274bd8b08cd1eda4b22553.c3b99209a6d4a5122f7ec5ad421b3c59->hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/0be85574bf274bd8b08cd1eda4b22553-top, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/8fb5c1122d9742efb84eca604960e857, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/TestLogRolling-testLogRolling=c3b99209a6d4a5122f7ec5ad421b3c59-9d90add7685f4c7497e834c4b7f7d4ed, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/e03145e5704b4f6183af391ae764dbff, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6547a6647a6c4823b65c0433f5c77410, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/b0af228d97014275b738a7bfc2c92789, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/bad3543619f0491e878c7e299091073c, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/2c87b271802f4abf9128b71c08d6665e, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/a0747b2f2c85495089d9258dc514adc8, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/19e142b9099c4544a2bf589ae1fda93d, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/137b89a25d7542eabb5863d900476ffd, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/d70e16e396d84399844f407b4b68010f, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/a591ac07fb24462380802621ab3837b3, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/9c65c2c378004c2996c7618bf5c5f84a, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6900c18cea4843918b21caa4ed5d57ac, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6a6ed6181c3a451dbd8f1765d1885a91, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/61006042e69e4f8d8e941434199f56ac, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/3d339ff028b6452ab06318443c6b794c, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/7aeaf42787984d59b5bb592ef0950cf8, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/c835bae53704450aaf0510c62d479deb, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/1ca878c1835b45fc858ab5a5c7a3c74b, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/adb91e48731d42dd8df4d3c61d73726e, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/9c6eb1b9271d474f8f84fbf85178f112, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/284b1e7c68a04a2dbc78e6cb6d52d18b, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/c61866c95490462cb105609f3a17d170, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/b7a273b98c6e43048b606aabbddb3762] to archive
2024-11-22T05:38:14,767 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-22T05:38:14,768 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/0be85574bf274bd8b08cd1eda4b22553.c3b99209a6d4a5122f7ec5ad421b3c59 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/0be85574bf274bd8b08cd1eda4b22553.c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:38:14,770 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-11-22T05:38:14,770 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/8fb5c1122d9742efb84eca604960e857 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/8fb5c1122d9742efb84eca604960e857
2024-11-22T05:38:14,770 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T05:38:14,770 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T05:38:14,770 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732253894765Running coprocessor pre-close hooks at 1732253894765Disabling compacts and flushes for region at 1732253894765Disabling writes for close at 1732253894765Writing region close event to WAL at 1732253894767 (+2 ms)Running coprocessor post-close hooks at 1732253894770 (+3 ms)Closed at 1732253894770
2024-11-22T05:38:14,771 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-22T05:38:14,771 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/TestLogRolling-testLogRolling=c3b99209a6d4a5122f7ec5ad421b3c59-9d90add7685f4c7497e834c4b7f7d4ed to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/TestLogRolling-testLogRolling=c3b99209a6d4a5122f7ec5ad421b3c59-9d90add7685f4c7497e834c4b7f7d4ed
2024-11-22T05:38:14,772 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/e03145e5704b4f6183af391ae764dbff to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/e03145e5704b4f6183af391ae764dbff
2024-11-22T05:38:14,773 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6547a6647a6c4823b65c0433f5c77410 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6547a6647a6c4823b65c0433f5c77410
2024-11-22T05:38:14,774 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/b0af228d97014275b738a7bfc2c92789 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/b0af228d97014275b738a7bfc2c92789
2024-11-22T05:38:14,776 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/bad3543619f0491e878c7e299091073c to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/bad3543619f0491e878c7e299091073c
2024-11-22T05:38:14,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:14,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:14,777 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/2c87b271802f4abf9128b71c08d6665e to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/2c87b271802f4abf9128b71c08d6665e
2024-11-22T05:38:14,778 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/a0747b2f2c85495089d9258dc514adc8 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/a0747b2f2c85495089d9258dc514adc8
2024-11-22T05:38:14,779 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/19e142b9099c4544a2bf589ae1fda93d to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/19e142b9099c4544a2bf589ae1fda93d
2024-11-22T05:38:14,780 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/137b89a25d7542eabb5863d900476ffd to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/137b89a25d7542eabb5863d900476ffd
2024-11-22T05:38:14,781 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/d70e16e396d84399844f407b4b68010f to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/d70e16e396d84399844f407b4b68010f
2024-11-22T05:38:14,782 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/a591ac07fb24462380802621ab3837b3 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/a591ac07fb24462380802621ab3837b3
2024-11-22T05:38:14,783 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/9c65c2c378004c2996c7618bf5c5f84a to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/9c65c2c378004c2996c7618bf5c5f84a
2024-11-22T05:38:14,784 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6900c18cea4843918b21caa4ed5d57ac to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6900c18cea4843918b21caa4ed5d57ac
2024-11-22T05:38:14,785 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6a6ed6181c3a451dbd8f1765d1885a91 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/6a6ed6181c3a451dbd8f1765d1885a91
2024-11-22T05:38:14,786 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/61006042e69e4f8d8e941434199f56ac to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/61006042e69e4f8d8e941434199f56ac
2024-11-22T05:38:14,786 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/3d339ff028b6452ab06318443c6b794c to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/3d339ff028b6452ab06318443c6b794c
2024-11-22T05:38:14,787 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/7aeaf42787984d59b5bb592ef0950cf8 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/7aeaf42787984d59b5bb592ef0950cf8
2024-11-22T05:38:14,788 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/c835bae53704450aaf0510c62d479deb to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/c835bae53704450aaf0510c62d479deb
2024-11-22T05:38:14,789 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/1ca878c1835b45fc858ab5a5c7a3c74b to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/1ca878c1835b45fc858ab5a5c7a3c74b
2024-11-22T05:38:14,790 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/adb91e48731d42dd8df4d3c61d73726e to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/adb91e48731d42dd8df4d3c61d73726e
2024-11-22T05:38:14,791 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/9c6eb1b9271d474f8f84fbf85178f112 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/9c6eb1b9271d474f8f84fbf85178f112
2024-11-22T05:38:14,792 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/284b1e7c68a04a2dbc78e6cb6d52d18b to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/284b1e7c68a04a2dbc78e6cb6d52d18b
2024-11-22T05:38:14,793 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/c61866c95490462cb105609f3a17d170 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/c61866c95490462cb105609f3a17d170
2024-11-22T05:38:14,794 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/b7a273b98c6e43048b606aabbddb3762 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/info/b7a273b98c6e43048b606aabbddb3762
2024-11-22T05:38:14,794 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=82891a793a29:44627 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-22T05:38:14,794 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [8fb5c1122d9742efb84eca604960e857=8359, e03145e5704b4f6183af391ae764dbff=12509, 6547a6647a6c4823b65c0433f5c77410=30029, b0af228d97014275b738a7bfc2c92789=18987, bad3543619f0491e878c7e299091073c=17895, 2c87b271802f4abf9128b71c08d6665e=50638, a0747b2f2c85495089d9258dc514adc8=12516, 19e142b9099c4544a2bf589ae1fda93d=20078, 137b89a25d7542eabb5863d900476ffd=76753, d70e16e396d84399844f407b4b68010f=15750, a591ac07fb24462380802621ab3837b3=12516, 9c65c2c378004c2996c7618bf5c5f84a=97341, 6900c18cea4843918b21caa4ed5d57ac=17906, 6a6ed6181c3a451dbd8f1765d1885a91=19000, 61006042e69e4f8d8e941434199f56ac=119007, 3d339ff028b6452ab06318443c6b794c=12516, 7aeaf42787984d59b5bb592ef0950cf8=21156, c835bae53704450aaf0510c62d479deb=147343, 1ca878c1835b45fc858ab5a5c7a3c74b=16829, adb91e48731d42dd8df4d3c61d73726e=12523, 9c6eb1b9271d474f8f84fbf85178f112=169029, 284b1e7c68a04a2dbc78e6cb6d52d18b=19013, c61866c95490462cb105609f3a17d170=19013, b7a273b98c6e43048b606aabbddb3762=15760]
2024-11-22T05:38:14,797 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/3fee3a70d358956b05e5e725f7751041/recovered.edits/346.seqid, newMaxSeqId=346, maxSeqId=88
2024-11-22T05:38:14,798 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:38:14,798 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3fee3a70d358956b05e5e725f7751041:
Waiting for close lock at 1732253894764Running coprocessor pre-close hooks at 1732253894764Disabling compacts and flushes for region at 1732253894764Disabling writes for close at 1732253894764Writing region close event to WAL at 1732253894795 (+31 ms)Running coprocessor post-close hooks at 1732253894798 (+3 ms)Closed at 1732253894798
2024-11-22T05:38:14,798 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1732253870000.3fee3a70d358956b05e5e725f7751041.
2024-11-22T05:38:14,798 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d7f712471bbedcd2472caa8e146dda67, disabling compactions & flushes
2024-11-22T05:38:14,798 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.
2024-11-22T05:38:14,798 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.
2024-11-22T05:38:14,798 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67. after waiting 0 ms
2024-11-22T05:38:14,798 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.
2024-11-22T05:38:14,799 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/d7f712471bbedcd2472caa8e146dda67/info/0be85574bf274bd8b08cd1eda4b22553.c3b99209a6d4a5122f7ec5ad421b3c59->hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/c3b99209a6d4a5122f7ec5ad421b3c59/info/0be85574bf274bd8b08cd1eda4b22553-bottom] to archive
2024-11-22T05:38:14,799 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-22T05:38:14,801 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/d7f712471bbedcd2472caa8e146dda67/info/0be85574bf274bd8b08cd1eda4b22553.c3b99209a6d4a5122f7ec5ad421b3c59 to hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/archive/data/default/TestLogRolling-testLogRolling/d7f712471bbedcd2472caa8e146dda67/info/0be85574bf274bd8b08cd1eda4b22553.c3b99209a6d4a5122f7ec5ad421b3c59
2024-11-22T05:38:14,801 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-11-22T05:38:14,804 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/data/default/TestLogRolling-testLogRolling/d7f712471bbedcd2472caa8e146dda67/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=88
2024-11-22T05:38:14,804 INFO  [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.
2024-11-22T05:38:14,804 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d7f712471bbedcd2472caa8e146dda67:
Waiting for close lock at 1732253894798Running coprocessor pre-close hooks at 1732253894798Disabling compacts and flushes for region at 1732253894798Disabling writes for close at 1732253894798Writing region close event to WAL at 1732253894801 (+3 ms)Running coprocessor post-close hooks at 1732253894804 (+3 ms)Closed at 1732253894804
2024-11-22T05:38:14,804 DEBUG [RS_CLOSE_REGION-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1732253870000.d7f712471bbedcd2472caa8e146dda67.
2024-11-22T05:38:14,965 INFO  [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(976): stopping server 82891a793a29,42063,1732253846222; all regions closed.
2024-11-22T05:38:14,966 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,966 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,966 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,966 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,966 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,969 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741834_1010 (size=8107)
2024-11-22T05:38:14,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741834_1010 (size=8107)
2024-11-22T05:38:14,974 DEBUG [RS:0;82891a793a29:42063 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/oldWALs
2024-11-22T05:38:14,974 INFO  [RS:0;82891a793a29:42063 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 82891a793a29%2C42063%2C1732253846222.meta:.meta(num 1732253847334)
2024-11-22T05:38:14,974 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,975 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,975 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,975 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,975 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:14,977 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741882_1058 (size=778)
2024-11-22T05:38:14,978 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741882_1058 (size=778)
2024-11-22T05:38:14,981 DEBUG [RS:0;82891a793a29:42063 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/oldWALs
2024-11-22T05:38:14,981 INFO  [RS:0;82891a793a29:42063 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 82891a793a29%2C42063%2C1732253846222:(num 1732253894580)
2024-11-22T05:38:14,981 DEBUG [RS:0;82891a793a29:42063 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:38:14,981 INFO  [RS:0;82891a793a29:42063 {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T05:38:14,981 INFO  [RS:0;82891a793a29:42063 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T05:38:14,981 INFO  [RS:0;82891a793a29:42063 {}] hbase.ChoreService(370): Chore service for: regionserver/82891a793a29:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-22T05:38:14,981 INFO  [RS:0;82891a793a29:42063 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T05:38:14,981 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T05:38:14,981 INFO  [RS:0;82891a793a29:42063 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42063
2024-11-22T05:38:14,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T05:38:14,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/82891a793a29,42063,1732253846222
2024-11-22T05:38:14,990 INFO  [RS:0;82891a793a29:42063 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T05:38:15,000 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [82891a793a29,42063,1732253846222]
2024-11-22T05:38:15,011 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/82891a793a29,42063,1732253846222 already deleted, retry=false
2024-11-22T05:38:15,011 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 82891a793a29,42063,1732253846222 expired; onlineServers=0
2024-11-22T05:38:15,011 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '82891a793a29,44627,1732253846045' *****
2024-11-22T05:38:15,011 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-22T05:38:15,011 INFO  [M:0;82891a793a29:44627 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T05:38:15,011 INFO  [M:0;82891a793a29:44627 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T05:38:15,011 DEBUG [M:0;82891a793a29:44627 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-22T05:38:15,011 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-22T05:38:15,011 DEBUG [M:0;82891a793a29:44627 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-22T05:38:15,011 DEBUG [master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253846572 {}] cleaner.HFileCleaner(306): Exit Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253846572,5,FailOnTimeoutGroup]
2024-11-22T05:38:15,011 DEBUG [master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253846571 {}] cleaner.HFileCleaner(306): Exit Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253846571,5,FailOnTimeoutGroup]
2024-11-22T05:38:15,011 INFO  [M:0;82891a793a29:44627 {}] hbase.ChoreService(370): Chore service for: master/82891a793a29:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-22T05:38:15,011 INFO  [M:0;82891a793a29:44627 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T05:38:15,011 DEBUG [M:0;82891a793a29:44627 {}] master.HMaster(1795): Stopping service threads
2024-11-22T05:38:15,011 INFO  [M:0;82891a793a29:44627 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-22T05:38:15,012 INFO  [M:0;82891a793a29:44627 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T05:38:15,012 INFO  [M:0;82891a793a29:44627 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-22T05:38:15,012 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-22T05:38:15,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-22T05:38:15,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:15,021 DEBUG [M:0;82891a793a29:44627 {}] zookeeper.ZKUtil(347): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-22T05:38:15,021 WARN  [M:0;82891a793a29:44627 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-22T05:38:15,022 INFO  [M:0;82891a793a29:44627 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/.lastflushedseqids
2024-11-22T05:38:15,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741883_1059 (size=228)
2024-11-22T05:38:15,031 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741883_1059 (size=228)
2024-11-22T05:38:15,031 INFO  [M:0;82891a793a29:44627 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-22T05:38:15,031 INFO  [M:0;82891a793a29:44627 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-22T05:38:15,031 DEBUG [M:0;82891a793a29:44627 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T05:38:15,031 INFO  [M:0;82891a793a29:44627 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:38:15,032 DEBUG [M:0;82891a793a29:44627 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:38:15,032 DEBUG [M:0;82891a793a29:44627 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T05:38:15,032 DEBUG [M:0;82891a793a29:44627 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:38:15,032 INFO  [M:0;82891a793a29:44627 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.42 KB heapSize=63.35 KB
2024-11-22T05:38:15,045 DEBUG [M:0;82891a793a29:44627 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b15c0e91b0394e8fa5e295cba521db6f is 82, key is hbase:meta,,1/info:regioninfo/1732253847383/Put/seqid=0
2024-11-22T05:38:15,049 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741884_1060 (size=5672)
2024-11-22T05:38:15,049 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741884_1060 (size=5672)
2024-11-22T05:38:15,050 INFO  [M:0;82891a793a29:44627 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b15c0e91b0394e8fa5e295cba521db6f
2024-11-22T05:38:15,066 DEBUG [M:0;82891a793a29:44627 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/285f7b6e4cea4a758d75afcbc72599b7 is 750, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732253847947/Put/seqid=0
2024-11-22T05:38:15,070 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741885_1061 (size=7090)
2024-11-22T05:38:15,070 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741885_1061 (size=7090)
2024-11-22T05:38:15,071 INFO  [M:0;82891a793a29:44627 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.81 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/285f7b6e4cea4a758d75afcbc72599b7
2024-11-22T05:38:15,074 INFO  [M:0;82891a793a29:44627 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 285f7b6e4cea4a758d75afcbc72599b7
2024-11-22T05:38:15,088 DEBUG [M:0;82891a793a29:44627 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0dacf63e8bec4c72b465cab4bc409081 is 69, key is 82891a793a29,42063,1732253846222/rs:state/1732253846689/Put/seqid=0
2024-11-22T05:38:15,091 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741886_1062 (size=5156)
2024-11-22T05:38:15,091 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741886_1062 (size=5156)
2024-11-22T05:38:15,092 INFO  [M:0;82891a793a29:44627 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0dacf63e8bec4c72b465cab4bc409081
2024-11-22T05:38:15,100 INFO  [RS:0;82891a793a29:42063 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T05:38:15,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:38:15,100 INFO  [RS:0;82891a793a29:42063 {}] regionserver.HRegionServer(1031): Exiting; stopping=82891a793a29,42063,1732253846222; zookeeper connection closed.
2024-11-22T05:38:15,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42063-0x101610c05ba0001, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:38:15,101 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@35bb88d2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@35bb88d2
2024-11-22T05:38:15,101 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-22T05:38:15,109 DEBUG [M:0;82891a793a29:44627 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f991aafcc28c40648330390de74862ca is 52, key is load_balancer_on/state:d/1732253847555/Put/seqid=0
2024-11-22T05:38:15,113 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741887_1063 (size=5056)
2024-11-22T05:38:15,113 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741887_1063 (size=5056)
2024-11-22T05:38:15,113 INFO  [M:0;82891a793a29:44627 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f991aafcc28c40648330390de74862ca
2024-11-22T05:38:15,118 DEBUG [M:0;82891a793a29:44627 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b15c0e91b0394e8fa5e295cba521db6f as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b15c0e91b0394e8fa5e295cba521db6f
2024-11-22T05:38:15,124 INFO  [M:0;82891a793a29:44627 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b15c0e91b0394e8fa5e295cba521db6f, entries=8, sequenceid=125, filesize=5.5 K
2024-11-22T05:38:15,125 DEBUG [M:0;82891a793a29:44627 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/285f7b6e4cea4a758d75afcbc72599b7 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/285f7b6e4cea4a758d75afcbc72599b7
2024-11-22T05:38:15,129 INFO  [M:0;82891a793a29:44627 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 285f7b6e4cea4a758d75afcbc72599b7
2024-11-22T05:38:15,129 INFO  [M:0;82891a793a29:44627 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/285f7b6e4cea4a758d75afcbc72599b7, entries=13, sequenceid=125, filesize=6.9 K
2024-11-22T05:38:15,130 DEBUG [M:0;82891a793a29:44627 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0dacf63e8bec4c72b465cab4bc409081 as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0dacf63e8bec4c72b465cab4bc409081
2024-11-22T05:38:15,135 INFO  [M:0;82891a793a29:44627 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0dacf63e8bec4c72b465cab4bc409081, entries=1, sequenceid=125, filesize=5.0 K
2024-11-22T05:38:15,136 DEBUG [M:0;82891a793a29:44627 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f991aafcc28c40648330390de74862ca as hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f991aafcc28c40648330390de74862ca
2024-11-22T05:38:15,140 INFO  [M:0;82891a793a29:44627 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41245/user/jenkins/test-data/df2d9c10-7358-6a42-7176-d216627f47da/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f991aafcc28c40648330390de74862ca, entries=1, sequenceid=125, filesize=4.9 K
2024-11-22T05:38:15,141 INFO  [M:0;82891a793a29:44627 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.42 KB/52651, heapSize ~63.29 KB/64808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 109ms, sequenceid=125, compaction requested=false
2024-11-22T05:38:15,144 INFO  [M:0;82891a793a29:44627 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:38:15,145 DEBUG [M:0;82891a793a29:44627 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732253895031Disabling compacts and flushes for region at 1732253895031Disabling writes for close at 1732253895032 (+1 ms)Obtaining lock to block concurrent updates at 1732253895032Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732253895032Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52651, getHeapSize=64808, getOffHeapSize=0, getCellsCount=148 at 1732253895032Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732253895033 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732253895033Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732253895045 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732253895045Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732253895054 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732253895066 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732253895066Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732253895075 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732253895088 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732253895088Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732253895095 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732253895108 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732253895108Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72445a8f: reopening flushed file at 1732253895117 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f68f693: reopening flushed file at 1732253895124 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c07ab1d: reopening flushed file at 1732253895130 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a6b1344: reopening flushed file at 1732253895135 (+5 ms)Finished flush of dataSize ~51.42 KB/52651, heapSize ~63.29 KB/64808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 109ms, sequenceid=125, compaction requested=false at 1732253895141 (+6 ms)Writing region close event to WAL at 1732253895144 (+3 ms)Closed at 1732253895144
2024-11-22T05:38:15,145 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:15,145 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:15,146 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:15,146 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:15,146 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:15,147 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35043 is added to blk_1073741830_1006 (size=61320)
2024-11-22T05:38:15,147 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39999 is added to blk_1073741830_1006 (size=61320)
2024-11-22T05:38:15,148 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T05:38:15,148 INFO  [M:0;82891a793a29:44627 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-22T05:38:15,148 INFO  [M:0;82891a793a29:44627 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44627
2024-11-22T05:38:15,148 INFO  [M:0;82891a793a29:44627 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T05:38:15,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:38:15,258 INFO  [M:0;82891a793a29:44627 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T05:38:15,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44627-0x101610c05ba0000, quorum=127.0.0.1:57233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:38:15,263 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@47bcda8c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:38:15,263 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@29765213{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:38:15,264 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:38:15,264 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@19fe8881{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:38:15,264 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@49490ce4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/hadoop.log.dir/,STOPPED}
2024-11-22T05:38:15,267 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:38:15,267 WARN  [BP-1931242728-172.17.0.2-1732253843348 heartbeating to localhost/127.0.0.1:41245 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:38:15,267 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:38:15,267 WARN  [BP-1931242728-172.17.0.2-1732253843348 heartbeating to localhost/127.0.0.1:41245 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1931242728-172.17.0.2-1732253843348 (Datanode Uuid d73574e7-bd8c-462d-9854-b1281436546b) service to localhost/127.0.0.1:41245
2024-11-22T05:38:15,269 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/cluster_48c49796-9abf-9856-dddf-c5191332e906/data/data3/current/BP-1931242728-172.17.0.2-1732253843348 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:38:15,269 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/cluster_48c49796-9abf-9856-dddf-c5191332e906/data/data4/current/BP-1931242728-172.17.0.2-1732253843348 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:38:15,269 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:38:15,271 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@23e1642c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:38:15,271 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@767f877d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:38:15,271 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:38:15,271 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@18d1ee92{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:38:15,271 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@354bdaa4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/hadoop.log.dir/,STOPPED}
2024-11-22T05:38:15,273 WARN  [BP-1931242728-172.17.0.2-1732253843348 heartbeating to localhost/127.0.0.1:41245 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:38:15,273 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:38:15,273 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:38:15,273 WARN  [BP-1931242728-172.17.0.2-1732253843348 heartbeating to localhost/127.0.0.1:41245 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1931242728-172.17.0.2-1732253843348 (Datanode Uuid 49d89928-7cc0-4701-83b0-5817393f9506) service to localhost/127.0.0.1:41245
2024-11-22T05:38:15,273 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/cluster_48c49796-9abf-9856-dddf-c5191332e906/data/data1/current/BP-1931242728-172.17.0.2-1732253843348 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:38:15,274 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/cluster_48c49796-9abf-9856-dddf-c5191332e906/data/data2/current/BP-1931242728-172.17.0.2-1732253843348 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:38:15,274 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:38:15,279 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@671b15e6{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T05:38:15,280 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2996c87f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:38:15,280 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:38:15,280 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@374d3611{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:38:15,280 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@79864455{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/hadoop.log.dir/,STOPPED}
2024-11-22T05:38:15,287 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-22T05:38:15,316 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-22T05:38:15,322 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=231 (was 206)
Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41245
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:41245
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41245 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41245 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41245
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:41245
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41245
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41245 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
 - Thread LEAK? -, OpenFileDescriptor=518 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 161), ProcessCount=11 (was 11), AvailableMemoryMB=7882 (was 7064) - AvailableMemoryMB LEAK? -
2024-11-22T05:38:15,329 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=231, OpenFileDescriptor=518, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=7882
2024-11-22T05:38:15,329 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-22T05:38:15,329 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/hadoop.log.dir so I do NOT create it in target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114
2024-11-22T05:38:15,330 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85e6540c-87cf-4296-b343-d071cebf5323/hadoop.tmp.dir so I do NOT create it in target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114
2024-11-22T05:38:15,330 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/cluster_0b2e7f0c-4f5f-a5c0-e78d-41c08b25443d, deleteOnExit=true
2024-11-22T05:38:15,330 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-22T05:38:15,330 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/test.cache.data in system properties and HBase conf
2024-11-22T05:38:15,330 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/hadoop.tmp.dir in system properties and HBase conf
2024-11-22T05:38:15,330 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/hadoop.log.dir in system properties and HBase conf
2024-11-22T05:38:15,330 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-22T05:38:15,330 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-22T05:38:15,330 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-22T05:38:15,330 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-22T05:38:15,330 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-22T05:38:15,330 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-22T05:38:15,330 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-22T05:38:15,330 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T05:38:15,330 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-22T05:38:15,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-22T05:38:15,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T05:38:15,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T05:38:15,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-22T05:38:15,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/nfs.dump.dir in system properties and HBase conf
2024-11-22T05:38:15,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/java.io.tmpdir in system properties and HBase conf
2024-11-22T05:38:15,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T05:38:15,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-22T05:38:15,331 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-22T05:38:15,344 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T05:38:15,699 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:38:15,702 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:38:15,703 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:38:15,703 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:38:15,703 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T05:38:15,704 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:38:15,704 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b01e479{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:38:15,704 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20042785{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:38:15,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:15,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:15,794 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4edfb46d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/java.io.tmpdir/jetty-localhost-43531-hadoop-hdfs-3_4_1-tests_jar-_-any-16146537480083705844/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T05:38:15,795 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7d6e4d22{HTTP/1.1, (http/1.1)}{localhost:43531}
2024-11-22T05:38:15,795 INFO  [Time-limited test {}] server.Server(415): Started @303052ms
2024-11-22T05:38:15,804 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T05:38:16,063 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:38:16,066 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:38:16,066 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:38:16,066 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:38:16,066 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T05:38:16,067 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@19f4d4a5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:38:16,067 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@512a083f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:38:16,160 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5dba136e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/java.io.tmpdir/jetty-localhost-37109-hadoop-hdfs-3_4_1-tests_jar-_-any-11090663303031440011/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:38:16,161 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e2fef96{HTTP/1.1, (http/1.1)}{localhost:37109}
2024-11-22T05:38:16,161 INFO  [Time-limited test {}] server.Server(415): Started @303418ms
2024-11-22T05:38:16,162 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:38:16,187 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T05:38:16,190 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T05:38:16,191 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T05:38:16,191 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T05:38:16,191 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T05:38:16,191 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@242dc355{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/hadoop.log.dir/,AVAILABLE}
2024-11-22T05:38:16,192 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1ba1b0df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T05:38:16,283 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2ea5aa4c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/java.io.tmpdir/jetty-localhost-33349-hadoop-hdfs-3_4_1-tests_jar-_-any-10663299898198893713/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:38:16,283 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@59622fd9{HTTP/1.1, (http/1.1)}{localhost:33349}
2024-11-22T05:38:16,283 INFO  [Time-limited test {}] server.Server(415): Started @303540ms
2024-11-22T05:38:16,284 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T05:38:16,721 INFO  [regionserver/82891a793a29:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T05:38:16,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:16,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:17,465 WARN  [Thread-2519 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/cluster_0b2e7f0c-4f5f-a5c0-e78d-41c08b25443d/data/data1/current/BP-220569439-172.17.0.2-1732253895346/current, will proceed with Du for space computation calculation, 
2024-11-22T05:38:17,465 WARN  [Thread-2520 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/cluster_0b2e7f0c-4f5f-a5c0-e78d-41c08b25443d/data/data2/current/BP-220569439-172.17.0.2-1732253895346/current, will proceed with Du for space computation calculation, 
2024-11-22T05:38:17,489 WARN  [Thread-2482 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:38:17,491 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe6c74530682d1d56 with lease ID 0x616e1722843dbb5c: Processing first storage report for DS-17829aa5-b724-469d-8d8a-3f50d68dacde from datanode DatanodeRegistration(127.0.0.1:38629, datanodeUuid=227a8f47-7bd0-41e9-93f2-524fc6c8cf3b, infoPort=38389, infoSecurePort=0, ipcPort=38853, storageInfo=lv=-57;cid=testClusterID;nsid=855548397;c=1732253895346)
2024-11-22T05:38:17,491 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe6c74530682d1d56 with lease ID 0x616e1722843dbb5c: from storage DS-17829aa5-b724-469d-8d8a-3f50d68dacde node DatanodeRegistration(127.0.0.1:38629, datanodeUuid=227a8f47-7bd0-41e9-93f2-524fc6c8cf3b, infoPort=38389, infoSecurePort=0, ipcPort=38853, storageInfo=lv=-57;cid=testClusterID;nsid=855548397;c=1732253895346), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:38:17,491 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe6c74530682d1d56 with lease ID 0x616e1722843dbb5c: Processing first storage report for DS-8dbd2e14-37ca-46a0-a4a5-7b305c72a72c from datanode DatanodeRegistration(127.0.0.1:38629, datanodeUuid=227a8f47-7bd0-41e9-93f2-524fc6c8cf3b, infoPort=38389, infoSecurePort=0, ipcPort=38853, storageInfo=lv=-57;cid=testClusterID;nsid=855548397;c=1732253895346)
2024-11-22T05:38:17,491 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe6c74530682d1d56 with lease ID 0x616e1722843dbb5c: from storage DS-8dbd2e14-37ca-46a0-a4a5-7b305c72a72c node DatanodeRegistration(127.0.0.1:38629, datanodeUuid=227a8f47-7bd0-41e9-93f2-524fc6c8cf3b, infoPort=38389, infoSecurePort=0, ipcPort=38853, storageInfo=lv=-57;cid=testClusterID;nsid=855548397;c=1732253895346), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:38:17,616 WARN  [Thread-2530 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/cluster_0b2e7f0c-4f5f-a5c0-e78d-41c08b25443d/data/data3/current/BP-220569439-172.17.0.2-1732253895346/current, will proceed with Du for space computation calculation, 
2024-11-22T05:38:17,616 WARN  [Thread-2531 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/cluster_0b2e7f0c-4f5f-a5c0-e78d-41c08b25443d/data/data4/current/BP-220569439-172.17.0.2-1732253895346/current, will proceed with Du for space computation calculation, 
2024-11-22T05:38:17,640 WARN  [Thread-2505 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T05:38:17,643 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x17fe8918004f94a2 with lease ID 0x616e1722843dbb5d: Processing first storage report for DS-6ea5157c-996b-42ec-80b9-e5c6405ae86c from datanode DatanodeRegistration(127.0.0.1:36843, datanodeUuid=aa18fedd-173b-405c-aae4-8777c145bda0, infoPort=45423, infoSecurePort=0, ipcPort=41831, storageInfo=lv=-57;cid=testClusterID;nsid=855548397;c=1732253895346)
2024-11-22T05:38:17,643 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x17fe8918004f94a2 with lease ID 0x616e1722843dbb5d: from storage DS-6ea5157c-996b-42ec-80b9-e5c6405ae86c node DatanodeRegistration(127.0.0.1:36843, datanodeUuid=aa18fedd-173b-405c-aae4-8777c145bda0, infoPort=45423, infoSecurePort=0, ipcPort=41831, storageInfo=lv=-57;cid=testClusterID;nsid=855548397;c=1732253895346), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:38:17,643 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x17fe8918004f94a2 with lease ID 0x616e1722843dbb5d: Processing first storage report for DS-72fca8cb-0106-46a9-b0bd-f05616f55760 from datanode DatanodeRegistration(127.0.0.1:36843, datanodeUuid=aa18fedd-173b-405c-aae4-8777c145bda0, infoPort=45423, infoSecurePort=0, ipcPort=41831, storageInfo=lv=-57;cid=testClusterID;nsid=855548397;c=1732253895346)
2024-11-22T05:38:17,643 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x17fe8918004f94a2 with lease ID 0x616e1722843dbb5d: from storage DS-72fca8cb-0106-46a9-b0bd-f05616f55760 node DatanodeRegistration(127.0.0.1:36843, datanodeUuid=aa18fedd-173b-405c-aae4-8777c145bda0, infoPort=45423, infoSecurePort=0, ipcPort=41831, storageInfo=lv=-57;cid=testClusterID;nsid=855548397;c=1732253895346), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T05:38:17,734 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114
2024-11-22T05:38:17,737 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/cluster_0b2e7f0c-4f5f-a5c0-e78d-41c08b25443d/zookeeper_0, clientPort=59322, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/cluster_0b2e7f0c-4f5f-a5c0-e78d-41c08b25443d/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/cluster_0b2e7f0c-4f5f-a5c0-e78d-41c08b25443d/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-22T05:38:17,739 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59322
2024-11-22T05:38:17,739 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:38:17,741 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:38:17,750 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741825_1001 (size=7)
2024-11-22T05:38:17,750 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741825_1001 (size=7)
2024-11-22T05:38:17,751 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4 with version=8
2024-11-22T05:38:17,751 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:40545/user/jenkins/test-data/16fb7876-2158-eddc-bf02-3d8a5f1fe26b/hbase-staging
2024-11-22T05:38:17,754 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/82891a793a29:0 server-side Connection retries=45
2024-11-22T05:38:17,754 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:38:17,754 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T05:38:17,754 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T05:38:17,754 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:38:17,754 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T05:38:17,754 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-22T05:38:17,754 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T05:38:17,755 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41489
2024-11-22T05:38:17,756 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41489 connecting to ZooKeeper ensemble=127.0.0.1:59322
2024-11-22T05:38:17,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:17,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:17,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:414890x0, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T05:38:17,983 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41489-0x101610ccfb40000 connected
2024-11-22T05:38:18,141 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:38:18,142 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:38:18,144 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:38:18,144 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4, hbase.cluster.distributed=false
2024-11-22T05:38:18,146 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T05:38:18,146 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41489
2024-11-22T05:38:18,146 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41489
2024-11-22T05:38:18,146 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41489
2024-11-22T05:38:18,147 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41489
2024-11-22T05:38:18,147 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41489
2024-11-22T05:38:18,161 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/82891a793a29:0 server-side Connection retries=45
2024-11-22T05:38:18,161 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:38:18,161 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T05:38:18,161 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T05:38:18,161 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T05:38:18,161 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T05:38:18,161 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-22T05:38:18,161 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T05:38:18,162 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33869
2024-11-22T05:38:18,163 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33869 connecting to ZooKeeper ensemble=127.0.0.1:59322
2024-11-22T05:38:18,163 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:38:18,165 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:38:18,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:338690x0, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T05:38:18,172 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33869-0x101610ccfb40001 connected
2024-11-22T05:38:18,172 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:38:18,173 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-22T05:38:18,173 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-22T05:38:18,174 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-22T05:38:18,175 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T05:38:18,175 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33869
2024-11-22T05:38:18,175 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33869
2024-11-22T05:38:18,176 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33869
2024-11-22T05:38:18,176 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33869
2024-11-22T05:38:18,176 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33869
2024-11-22T05:38:18,189 DEBUG [M:0;82891a793a29:41489 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;82891a793a29:41489
2024-11-22T05:38:18,189 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/82891a793a29,41489,1732253897753
2024-11-22T05:38:18,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:38:18,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:38:18,202 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/82891a793a29,41489,1732253897753
2024-11-22T05:38:18,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-22T05:38:18,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:18,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:18,215 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-22T05:38:18,216 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/82891a793a29,41489,1732253897753 from backup master directory
2024-11-22T05:38:18,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/82891a793a29,41489,1732253897753
2024-11-22T05:38:18,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:38:18,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T05:38:18,225 WARN  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T05:38:18,225 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=82891a793a29,41489,1732253897753
2024-11-22T05:38:18,232 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/hbase.id] with ID: 26d16c25-535c-4122-b1a8-ddffd67ab1d1
2024-11-22T05:38:18,232 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/.tmp/hbase.id
2024-11-22T05:38:18,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741826_1002 (size=42)
2024-11-22T05:38:18,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741826_1002 (size=42)
2024-11-22T05:38:18,242 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/.tmp/hbase.id]:[hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/hbase.id]
2024-11-22T05:38:18,252 INFO  [master/82891a793a29:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:38:18,252 INFO  [master/82891a793a29:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-22T05:38:18,254 INFO  [master/82891a793a29:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-22T05:38:18,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:18,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:18,272 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741827_1003 (size=196)
2024-11-22T05:38:18,272 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741827_1003 (size=196)
2024-11-22T05:38:18,273 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T05:38:18,274 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-22T05:38:18,274 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:38:18,280 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741828_1004 (size=1189)
2024-11-22T05:38:18,280 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741828_1004 (size=1189)
2024-11-22T05:38:18,281 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store
2024-11-22T05:38:18,286 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741829_1005 (size=34)
2024-11-22T05:38:18,286 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741829_1005 (size=34)
2024-11-22T05:38:18,286 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:38:18,286 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T05:38:18,287 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:38:18,287 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:38:18,287 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T05:38:18,287 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:38:18,287 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:38:18,287 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732253898286Disabling compacts and flushes for region at 1732253898286Disabling writes for close at 1732253898287 (+1 ms)Writing region close event to WAL at 1732253898287Closed at 1732253898287
2024-11-22T05:38:18,287 WARN  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/.initializing
2024-11-22T05:38:18,287 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/WALs/82891a793a29,41489,1732253897753
2024-11-22T05:38:18,289 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C41489%2C1732253897753, suffix=, logDir=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/WALs/82891a793a29,41489,1732253897753, archiveDir=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/oldWALs, maxLogs=10
2024-11-22T05:38:18,290 INFO  [master/82891a793a29:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C41489%2C1732253897753.1732253898290
2024-11-22T05:38:18,296 INFO  [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/WALs/82891a793a29,41489,1732253897753/82891a793a29%2C41489%2C1732253897753.1732253898290
2024-11-22T05:38:18,300 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38389:38389),(127.0.0.1/127.0.0.1:45423:45423)]
2024-11-22T05:38:18,301 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:38:18,301 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:38:18,301 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:38:18,301 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:38:18,302 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:38:18,303 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-22T05:38:18,303 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:38:18,304 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:38:18,304 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:38:18,305 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-22T05:38:18,305 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:38:18,305 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:38:18,305 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:38:18,307 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-22T05:38:18,307 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:38:18,307 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:38:18,307 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T05:38:18,309 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-22T05:38:18,309 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:38:18,309 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T05:38:18,309 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:38:18,310 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:38:18,310 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:38:18,311 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:38:18,311 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:38:18,312 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-22T05:38:18,313 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T05:38:18,314 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:38:18,315 INFO  [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=826456, jitterRate=0.050893738865852356}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-22T05:38:18,315 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732253898301Initializing all the Stores at 1732253898302 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253898302Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253898302Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253898302Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253898302Cleaning up temporary data from old regions at 1732253898311 (+9 ms)Region opened successfully at 1732253898315 (+4 ms)
2024-11-22T05:38:18,315 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-22T05:38:18,318 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@376d3665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82891a793a29/172.17.0.2:0
2024-11-22T05:38:18,318 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-22T05:38:18,318 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-22T05:38:18,319 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-22T05:38:18,319 INFO  [master/82891a793a29:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-22T05:38:18,319 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-22T05:38:18,319 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-22T05:38:18,319 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-22T05:38:18,321 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-22T05:38:18,322 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-22T05:38:18,330 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-22T05:38:18,330 INFO  [master/82891a793a29:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-22T05:38:18,330 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-22T05:38:18,419 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-22T05:38:18,419 INFO  [master/82891a793a29:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-22T05:38:18,420 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-22T05:38:18,446 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-22T05:38:18,447 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-22T05:38:18,456 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-22T05:38:18,460 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-22T05:38:18,467 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-22T05:38:18,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T05:38:18,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T05:38:18,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:18,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:18,478 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=82891a793a29,41489,1732253897753, sessionid=0x101610ccfb40000, setting cluster-up flag (Was=false)
2024-11-22T05:38:18,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:18,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:18,530 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-22T05:38:18,531 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=82891a793a29,41489,1732253897753
2024-11-22T05:38:18,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:18,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:18,582 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-22T05:38:18,584 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=82891a793a29,41489,1732253897753
2024-11-22T05:38:18,585 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-22T05:38:18,587 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-22T05:38:18,587 INFO  [master/82891a793a29:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-22T05:38:18,587 INFO  [master/82891a793a29:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-22T05:38:18,587 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 82891a793a29,41489,1732253897753
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-22T05:38:18,589 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:38:18,589 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:38:18,589 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:38:18,589 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/82891a793a29:0, corePoolSize=5, maxPoolSize=5
2024-11-22T05:38:18,589 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/82891a793a29:0, corePoolSize=10, maxPoolSize=10
2024-11-22T05:38:18,589 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:38:18,589 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/82891a793a29:0, corePoolSize=2, maxPoolSize=2
2024-11-22T05:38:18,589 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:38:18,591 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:38:18,591 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-22T05:38:18,591 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732253928591
2024-11-22T05:38:18,591 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-22T05:38:18,591 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-22T05:38:18,592 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-22T05:38:18,592 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-22T05:38:18,592 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-22T05:38:18,592 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-22T05:38:18,592 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:18,592 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-22T05:38:18,592 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:38:18,592 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-22T05:38:18,592 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-22T05:38:18,592 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-22T05:38:18,593 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-22T05:38:18,593 INFO  [master/82891a793a29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-22T05:38:18,593 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253898593,5,FailOnTimeoutGroup]
2024-11-22T05:38:18,594 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253898593,5,FailOnTimeoutGroup]
2024-11-22T05:38:18,594 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:18,594 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-22T05:38:18,594 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:18,594 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:18,598 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741831_1007 (size=1321)
2024-11-22T05:38:18,598 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741831_1007 (size=1321)
2024-11-22T05:38:18,598 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-22T05:38:18,598 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4
2024-11-22T05:38:18,603 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741832_1008 (size=32)
2024-11-22T05:38:18,603 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741832_1008 (size=32)
2024-11-22T05:38:18,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:38:18,605 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T05:38:18,606 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T05:38:18,606 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:38:18,606 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:38:18,606 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T05:38:18,607 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T05:38:18,607 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:38:18,607 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:38:18,607 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T05:38:18,608 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T05:38:18,608 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:38:18,608 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:38:18,608 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T05:38:18,609 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T05:38:18,610 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:38:18,610 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:38:18,610 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T05:38:18,610 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/data/hbase/meta/1588230740
2024-11-22T05:38:18,611 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/data/hbase/meta/1588230740
2024-11-22T05:38:18,612 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T05:38:18,612 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T05:38:18,612 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T05:38:18,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T05:38:18,614 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T05:38:18,615 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=787146, jitterRate=9.088367223739624E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T05:38:18,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732253898604Initializing all the Stores at 1732253898604Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253898604Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253898604Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253898604Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253898604Cleaning up temporary data from old regions at 1732253898612 (+8 ms)Region opened successfully at 1732253898615 (+3 ms)
2024-11-22T05:38:18,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T05:38:18,615 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T05:38:18,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T05:38:18,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T05:38:18,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T05:38:18,616 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T05:38:18,616 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732253898615Disabling compacts and flushes for region at 1732253898615Disabling writes for close at 1732253898615Writing region close event to WAL at 1732253898616 (+1 ms)Closed at 1732253898616
2024-11-22T05:38:18,617 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:38:18,617 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-22T05:38:18,617 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-22T05:38:18,618 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T05:38:18,619 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-22T05:38:18,678 INFO  [RS:0;82891a793a29:33869 {}] regionserver.HRegionServer(746): ClusterId : 26d16c25-535c-4122-b1a8-ddffd67ab1d1
2024-11-22T05:38:18,678 DEBUG [RS:0;82891a793a29:33869 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-22T05:38:18,699 DEBUG [RS:0;82891a793a29:33869 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-22T05:38:18,699 DEBUG [RS:0;82891a793a29:33869 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-22T05:38:18,710 DEBUG [RS:0;82891a793a29:33869 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-22T05:38:18,710 DEBUG [RS:0;82891a793a29:33869 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d9a01a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82891a793a29/172.17.0.2:0
2024-11-22T05:38:18,723 DEBUG [RS:0;82891a793a29:33869 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;82891a793a29:33869
2024-11-22T05:38:18,723 INFO  [RS:0;82891a793a29:33869 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-22T05:38:18,723 INFO  [RS:0;82891a793a29:33869 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-22T05:38:18,723 DEBUG [RS:0;82891a793a29:33869 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-22T05:38:18,724 INFO  [RS:0;82891a793a29:33869 {}] regionserver.HRegionServer(2659): reportForDuty to master=82891a793a29,41489,1732253897753 with port=33869, startcode=1732253898160
2024-11-22T05:38:18,724 DEBUG [RS:0;82891a793a29:33869 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-22T05:38:18,726 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35781, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-11-22T05:38:18,727 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41489 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 82891a793a29,33869,1732253898160
2024-11-22T05:38:18,727 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41489 {}] master.ServerManager(517): Registering regionserver=82891a793a29,33869,1732253898160
2024-11-22T05:38:18,728 DEBUG [RS:0;82891a793a29:33869 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4
2024-11-22T05:38:18,728 DEBUG [RS:0;82891a793a29:33869 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46055
2024-11-22T05:38:18,728 DEBUG [RS:0;82891a793a29:33869 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-22T05:38:18,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T05:38:18,741 DEBUG [RS:0;82891a793a29:33869 {}] zookeeper.ZKUtil(111): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/82891a793a29,33869,1732253898160
2024-11-22T05:38:18,741 WARN  [RS:0;82891a793a29:33869 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T05:38:18,741 INFO  [RS:0;82891a793a29:33869 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:38:18,741 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [82891a793a29,33869,1732253898160]
2024-11-22T05:38:18,741 DEBUG [RS:0;82891a793a29:33869 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/WALs/82891a793a29,33869,1732253898160
2024-11-22T05:38:18,745 INFO  [RS:0;82891a793a29:33869 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-22T05:38:18,746 INFO  [RS:0;82891a793a29:33869 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-22T05:38:18,746 INFO  [RS:0;82891a793a29:33869 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T05:38:18,746 INFO  [RS:0;82891a793a29:33869 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:18,747 INFO  [RS:0;82891a793a29:33869 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-22T05:38:18,747 INFO  [RS:0;82891a793a29:33869 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-22T05:38:18,747 INFO  [RS:0;82891a793a29:33869 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:18,748 DEBUG [RS:0;82891a793a29:33869 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:38:18,748 DEBUG [RS:0;82891a793a29:33869 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:38:18,748 DEBUG [RS:0;82891a793a29:33869 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:38:18,748 DEBUG [RS:0;82891a793a29:33869 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:38:18,748 DEBUG [RS:0;82891a793a29:33869 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:38:18,748 DEBUG [RS:0;82891a793a29:33869 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/82891a793a29:0, corePoolSize=2, maxPoolSize=2
2024-11-22T05:38:18,748 DEBUG [RS:0;82891a793a29:33869 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:38:18,748 DEBUG [RS:0;82891a793a29:33869 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:38:18,748 DEBUG [RS:0;82891a793a29:33869 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:38:18,748 DEBUG [RS:0;82891a793a29:33869 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:38:18,748 DEBUG [RS:0;82891a793a29:33869 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:38:18,748 DEBUG [RS:0;82891a793a29:33869 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/82891a793a29:0, corePoolSize=1, maxPoolSize=1
2024-11-22T05:38:18,748 DEBUG [RS:0;82891a793a29:33869 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/82891a793a29:0, corePoolSize=3, maxPoolSize=3
2024-11-22T05:38:18,748 DEBUG [RS:0;82891a793a29:33869 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/82891a793a29:0, corePoolSize=3, maxPoolSize=3
2024-11-22T05:38:18,750 INFO  [RS:0;82891a793a29:33869 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:18,750 INFO  [RS:0;82891a793a29:33869 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:18,750 INFO  [RS:0;82891a793a29:33869 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:18,750 INFO  [RS:0;82891a793a29:33869 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:18,750 INFO  [RS:0;82891a793a29:33869 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:18,750 INFO  [RS:0;82891a793a29:33869 {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,33869,1732253898160-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T05:38:18,768 INFO  [RS:0;82891a793a29:33869 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-22T05:38:18,768 INFO  [RS:0;82891a793a29:33869 {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,33869,1732253898160-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:18,768 INFO  [RS:0;82891a793a29:33869 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:18,768 INFO  [RS:0;82891a793a29:33869 {}] regionserver.Replication(171): 82891a793a29,33869,1732253898160 started
2024-11-22T05:38:18,769 WARN  [82891a793a29:41489 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-22T05:38:18,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:18,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:18,783 INFO  [RS:0;82891a793a29:33869 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:18,783 INFO  [RS:0;82891a793a29:33869 {}] regionserver.HRegionServer(1482): Serving as 82891a793a29,33869,1732253898160, RpcServer on 82891a793a29/172.17.0.2:33869, sessionid=0x101610ccfb40001
2024-11-22T05:38:18,783 DEBUG [RS:0;82891a793a29:33869 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-22T05:38:18,783 DEBUG [RS:0;82891a793a29:33869 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 82891a793a29,33869,1732253898160
2024-11-22T05:38:18,783 DEBUG [RS:0;82891a793a29:33869 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82891a793a29,33869,1732253898160'
2024-11-22T05:38:18,783 DEBUG [RS:0;82891a793a29:33869 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-22T05:38:18,784 DEBUG [RS:0;82891a793a29:33869 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-22T05:38:18,784 DEBUG [RS:0;82891a793a29:33869 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-22T05:38:18,784 DEBUG [RS:0;82891a793a29:33869 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-22T05:38:18,784 DEBUG [RS:0;82891a793a29:33869 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 82891a793a29,33869,1732253898160
2024-11-22T05:38:18,784 DEBUG [RS:0;82891a793a29:33869 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82891a793a29,33869,1732253898160'
2024-11-22T05:38:18,784 DEBUG [RS:0;82891a793a29:33869 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-22T05:38:18,785 DEBUG [RS:0;82891a793a29:33869 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-22T05:38:18,785 DEBUG [RS:0;82891a793a29:33869 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-22T05:38:18,785 INFO  [RS:0;82891a793a29:33869 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-22T05:38:18,785 INFO  [RS:0;82891a793a29:33869 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-22T05:38:18,889 INFO  [RS:0;82891a793a29:33869 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C33869%2C1732253898160, suffix=, logDir=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/WALs/82891a793a29,33869,1732253898160, archiveDir=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/oldWALs, maxLogs=32
2024-11-22T05:38:18,889 INFO  [RS:0;82891a793a29:33869 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C33869%2C1732253898160.1732253898889
2024-11-22T05:38:18,897 INFO  [RS:0;82891a793a29:33869 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/WALs/82891a793a29,33869,1732253898160/82891a793a29%2C33869%2C1732253898160.1732253898889
2024-11-22T05:38:18,902 DEBUG [RS:0;82891a793a29:33869 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38389:38389),(127.0.0.1/127.0.0.1:45423:45423)]
2024-11-22T05:38:19,019 DEBUG [82891a793a29:41489 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-22T05:38:19,020 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=82891a793a29,33869,1732253898160
2024-11-22T05:38:19,021 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 82891a793a29,33869,1732253898160, state=OPENING
2024-11-22T05:38:19,032 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-22T05:38:19,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:19,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:19,044 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T05:38:19,044 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:38:19,044 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:38:19,044 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=82891a793a29,33869,1732253898160}]
2024-11-22T05:38:19,196 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-22T05:38:19,199 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57841, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-22T05:38:19,204 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-22T05:38:19,204 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:38:19,207 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82891a793a29%2C33869%2C1732253898160.meta, suffix=.meta, logDir=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/WALs/82891a793a29,33869,1732253898160, archiveDir=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/oldWALs, maxLogs=32
2024-11-22T05:38:19,208 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 82891a793a29%2C33869%2C1732253898160.meta.1732253899208.meta
2024-11-22T05:38:19,213 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/WALs/82891a793a29,33869,1732253898160/82891a793a29%2C33869%2C1732253898160.meta.1732253899208.meta
2024-11-22T05:38:19,216 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45423:45423),(127.0.0.1/127.0.0.1:38389:38389)]
2024-11-22T05:38:19,220 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-22T05:38:19,220 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-22T05:38:19,220 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-22T05:38:19,221 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-22T05:38:19,221 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-22T05:38:19,221 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T05:38:19,221 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-22T05:38:19,221 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-22T05:38:19,223 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T05:38:19,224 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T05:38:19,224 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:38:19,225 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:38:19,225 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T05:38:19,225 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T05:38:19,225 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:38:19,226 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:38:19,226 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T05:38:19,226 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T05:38:19,226 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:38:19,227 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:38:19,227 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T05:38:19,227 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T05:38:19,227 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T05:38:19,227 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T05:38:19,228 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T05:38:19,228 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/data/hbase/meta/1588230740
2024-11-22T05:38:19,229 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/data/hbase/meta/1588230740
2024-11-22T05:38:19,230 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T05:38:19,230 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T05:38:19,231 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T05:38:19,232 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T05:38:19,233 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=842332, jitterRate=0.07108074426651001}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T05:38:19,233 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-22T05:38:19,233 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732253899221Writing region info on filesystem at 1732253899221Initializing all the Stores at 1732253899222 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253899222Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253899223 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732253899223Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732253899223Cleaning up temporary data from old regions at 1732253899230 (+7 ms)Running coprocessor post-open hooks at 1732253899233 (+3 ms)Region opened successfully at 1732253899233
2024-11-22T05:38:19,234 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732253899196
2024-11-22T05:38:19,236 DEBUG [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-22T05:38:19,236 INFO  [RS_OPEN_META-regionserver/82891a793a29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-22T05:38:19,237 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=82891a793a29,33869,1732253898160
2024-11-22T05:38:19,237 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 82891a793a29,33869,1732253898160, state=OPEN
2024-11-22T05:38:19,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T05:38:19,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T05:38:19,279 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=82891a793a29,33869,1732253898160
2024-11-22T05:38:19,279 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:38:19,279 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T05:38:19,282 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-22T05:38:19,282 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=82891a793a29,33869,1732253898160 in 235 msec
2024-11-22T05:38:19,286 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-22T05:38:19,286 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 665 msec
2024-11-22T05:38:19,287 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T05:38:19,287 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-22T05:38:19,289 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T05:38:19,289 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82891a793a29,33869,1732253898160, seqNum=-1]
2024-11-22T05:38:19,290 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T05:38:19,292 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37665, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T05:38:19,297 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 710 msec
2024-11-22T05:38:19,297 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732253899297, completionTime=-1
2024-11-22T05:38:19,297 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-22T05:38:19,297 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-22T05:38:19,299 INFO  [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-22T05:38:19,299 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732253959299
2024-11-22T05:38:19,299 INFO  [master/82891a793a29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732254019299
2024-11-22T05:38:19,299 INFO  [master/82891a793a29:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-22T05:38:19,300 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,41489,1732253897753-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:19,300 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,41489,1732253897753-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:19,300 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,41489,1732253897753-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:19,300 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-82891a793a29:41489, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:19,300 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:19,300 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:19,301 DEBUG [master/82891a793a29:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-22T05:38:19,304 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.079sec
2024-11-22T05:38:19,304 INFO  [master/82891a793a29:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-22T05:38:19,304 INFO  [master/82891a793a29:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-22T05:38:19,304 INFO  [master/82891a793a29:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-22T05:38:19,304 INFO  [master/82891a793a29:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-22T05:38:19,304 INFO  [master/82891a793a29:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-22T05:38:19,304 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,41489,1732253897753-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T05:38:19,304 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,41489,1732253897753-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-22T05:38:19,306 DEBUG [master/82891a793a29:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-22T05:38:19,306 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-22T05:38:19,306 INFO  [master/82891a793a29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82891a793a29,41489,1732253897753-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T05:38:19,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@665a6083, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:38:19,379 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82891a793a29,41489,-1 for getting cluster id
2024-11-22T05:38:19,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-22T05:38:19,381 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26d16c25-535c-4122-b1a8-ddffd67ab1d1'
2024-11-22T05:38:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-22T05:38:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26d16c25-535c-4122-b1a8-ddffd67ab1d1"

2024-11-22T05:38:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57e92c2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:38:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82891a793a29,41489,-1]
2024-11-22T05:38:19,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-22T05:38:19,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:38:19,383 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-22T05:38:19,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c6a0fbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T05:38:19,384 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T05:38:19,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82891a793a29,33869,1732253898160, seqNum=-1]
2024-11-22T05:38:19,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T05:38:19,386 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T05:38:19,388 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=82891a793a29,41489,1732253897753
2024-11-22T05:38:19,388 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T05:38:19,391 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-22T05:38:19,391 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T05:38:19,394 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/WALs/test.com,8080,1, archiveDir=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/oldWALs, maxLogs=32
2024-11-22T05:38:19,394 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732253899394
2024-11-22T05:38:19,400 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/WALs/test.com,8080,1/test.com%2C8080%2C1.1732253899394
2024-11-22T05:38:19,401 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45423:45423),(127.0.0.1/127.0.0.1:38389:38389)]
2024-11-22T05:38:19,402 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732253899402
2024-11-22T05:38:19,409 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,409 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,410 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,410 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,410 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,410 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/WALs/test.com,8080,1/test.com%2C8080%2C1.1732253899394 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/WALs/test.com,8080,1/test.com%2C8080%2C1.1732253899402
2024-11-22T05:38:19,411 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45423:45423),(127.0.0.1/127.0.0.1:38389:38389)]
2024-11-22T05:38:19,411 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/WALs/test.com,8080,1/test.com%2C8080%2C1.1732253899394 is not closed yet, will try archiving it next time
2024-11-22T05:38:19,412 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,412 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741835_1011 (size=93)
2024-11-22T05:38:19,412 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,412 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,412 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741835_1011 (size=93)
2024-11-22T05:38:19,413 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,413 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,414 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/WALs/test.com,8080,1/test.com%2C8080%2C1.1732253899394 to hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/oldWALs/test.com%2C8080%2C1.1732253899394
2024-11-22T05:38:19,415 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741836_1012 (size=93)
2024-11-22T05:38:19,415 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741836_1012 (size=93)
2024-11-22T05:38:19,418 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/oldWALs
2024-11-22T05:38:19,418 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1732253899402)
2024-11-22T05:38:19,418 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-22T05:38:19,418 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T05:38:19,418 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:38:19,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:38:19,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:38:19,418 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-22T05:38:19,418 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-22T05:38:19,418 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1938202520, stopped=false
2024-11-22T05:38:19,418 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=82891a793a29,41489,1732253897753
2024-11-22T05:38:19,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T05:38:19,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T05:38:19,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:19,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:19,435 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T05:38:19,436 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T05:38:19,436 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:38:19,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:38:19,436 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:38:19,436 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '82891a793a29,33869,1732253898160' *****
2024-11-22T05:38:19,436 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-22T05:38:19,436 INFO  [RS:0;82891a793a29:33869 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-22T05:38:19,437 INFO  [RS:0;82891a793a29:33869 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-22T05:38:19,437 INFO  [RS:0;82891a793a29:33869 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-22T05:38:19,437 INFO  [RS:0;82891a793a29:33869 {}] regionserver.HRegionServer(959): stopping server 82891a793a29,33869,1732253898160
2024-11-22T05:38:19,437 INFO  [RS:0;82891a793a29:33869 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T05:38:19,437 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-22T05:38:19,437 INFO  [RS:0;82891a793a29:33869 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;82891a793a29:33869.
2024-11-22T05:38:19,437 DEBUG [RS:0;82891a793a29:33869 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T05:38:19,437 DEBUG [RS:0;82891a793a29:33869 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:38:19,437 INFO  [RS:0;82891a793a29:33869 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-22T05:38:19,437 INFO  [RS:0;82891a793a29:33869 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-22T05:38:19,437 INFO  [RS:0;82891a793a29:33869 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-22T05:38:19,437 INFO  [RS:0;82891a793a29:33869 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-22T05:38:19,437 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T05:38:19,438 INFO  [RS:0;82891a793a29:33869 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-22T05:38:19,438 DEBUG [RS:0;82891a793a29:33869 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-22T05:38:19,438 DEBUG [RS:0;82891a793a29:33869 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-22T05:38:19,439 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T05:38:19,439 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T05:38:19,439 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T05:38:19,439 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T05:38:19,439 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T05:38:19,439 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-22T05:38:19,456 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/data/hbase/meta/1588230740/.tmp/ns/406f208c95ed42219b8f50b617002626 is 43, key is default/ns:d/1732253899292/Put/seqid=0
2024-11-22T05:38:19,460 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741837_1013 (size=5153)
2024-11-22T05:38:19,461 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741837_1013 (size=5153)
2024-11-22T05:38:19,461 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/data/hbase/meta/1588230740/.tmp/ns/406f208c95ed42219b8f50b617002626
2024-11-22T05:38:19,467 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/data/hbase/meta/1588230740/.tmp/ns/406f208c95ed42219b8f50b617002626 as hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/data/hbase/meta/1588230740/ns/406f208c95ed42219b8f50b617002626
2024-11-22T05:38:19,472 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/data/hbase/meta/1588230740/ns/406f208c95ed42219b8f50b617002626, entries=2, sequenceid=6, filesize=5.0 K
2024-11-22T05:38:19,473 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 34ms, sequenceid=6, compaction requested=false
2024-11-22T05:38:19,477 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-22T05:38:19,478 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T05:38:19,478 INFO  [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T05:38:19,478 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732253899438Running coprocessor pre-close hooks at 1732253899438Disabling compacts and flushes for region at 1732253899438Disabling writes for close at 1732253899439 (+1 ms)Obtaining lock to block concurrent updates at 1732253899439Preparing flush snapshotting stores in 1588230740 at 1732253899439Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732253899439Flushing stores of hbase:meta,,1.1588230740 at 1732253899440 (+1 ms)Flushing 1588230740/ns: creating writer at 1732253899440Flushing 1588230740/ns: appending metadata at 1732253899456 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732253899456Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d429e1: reopening flushed file at 1732253899466 (+10 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 34ms, sequenceid=6, compaction requested=false at 1732253899473 (+7 ms)Writing region close event to WAL at 1732253899473Running coprocessor post-close hooks at 1732253899478 (+5 ms)Closed at 1732253899478
2024-11-22T05:38:19,478 DEBUG [RS_CLOSE_META-regionserver/82891a793a29:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-22T05:38:19,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T05:38:19,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-22T05:38:19,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-22T05:38:19,639 INFO  [RS:0;82891a793a29:33869 {}] regionserver.HRegionServer(976): stopping server 82891a793a29,33869,1732253898160; all regions closed.
2024-11-22T05:38:19,640 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,640 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,640 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,640 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,640 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,643 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741834_1010 (size=1152)
2024-11-22T05:38:19,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741834_1010 (size=1152)
2024-11-22T05:38:19,647 DEBUG [RS:0;82891a793a29:33869 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/oldWALs
2024-11-22T05:38:19,647 INFO  [RS:0;82891a793a29:33869 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 82891a793a29%2C33869%2C1732253898160.meta:.meta(num 1732253899208)
2024-11-22T05:38:19,647 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,647 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,647 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,647 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,647 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741833_1009 (size=93)
2024-11-22T05:38:19,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741833_1009 (size=93)
2024-11-22T05:38:19,653 DEBUG [RS:0;82891a793a29:33869 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/oldWALs
2024-11-22T05:38:19,653 INFO  [RS:0;82891a793a29:33869 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 82891a793a29%2C33869%2C1732253898160:(num 1732253898889)
2024-11-22T05:38:19,653 DEBUG [RS:0;82891a793a29:33869 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T05:38:19,653 INFO  [RS:0;82891a793a29:33869 {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T05:38:19,653 INFO  [RS:0;82891a793a29:33869 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T05:38:19,653 INFO  [RS:0;82891a793a29:33869 {}] hbase.ChoreService(370): Chore service for: regionserver/82891a793a29:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-22T05:38:19,653 INFO  [RS:0;82891a793a29:33869 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T05:38:19,653 INFO  [regionserver/82891a793a29:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T05:38:19,653 INFO  [RS:0;82891a793a29:33869 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33869
2024-11-22T05:38:19,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T05:38:19,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/82891a793a29,33869,1732253898160
2024-11-22T05:38:19,664 INFO  [RS:0;82891a793a29:33869 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T05:38:19,675 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [82891a793a29,33869,1732253898160]
2024-11-22T05:38:19,685 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/82891a793a29,33869,1732253898160 already deleted, retry=false
2024-11-22T05:38:19,685 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 82891a793a29,33869,1732253898160 expired; onlineServers=0
2024-11-22T05:38:19,685 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '82891a793a29,41489,1732253897753' *****
2024-11-22T05:38:19,685 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-22T05:38:19,685 INFO  [M:0;82891a793a29:41489 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T05:38:19,685 INFO  [M:0;82891a793a29:41489 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T05:38:19,685 DEBUG [M:0;82891a793a29:41489 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-22T05:38:19,686 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-22T05:38:19,686 DEBUG [M:0;82891a793a29:41489 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-22T05:38:19,686 DEBUG [master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253898593 {}] cleaner.HFileCleaner(306): Exit Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.small.0-1732253898593,5,FailOnTimeoutGroup]
2024-11-22T05:38:19,686 DEBUG [master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253898593 {}] cleaner.HFileCleaner(306): Exit Thread[master/82891a793a29:0:becomeActiveMaster-HFileCleaner.large.0-1732253898593,5,FailOnTimeoutGroup]
2024-11-22T05:38:19,686 INFO  [M:0;82891a793a29:41489 {}] hbase.ChoreService(370): Chore service for: master/82891a793a29:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-22T05:38:19,686 INFO  [M:0;82891a793a29:41489 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T05:38:19,686 DEBUG [M:0;82891a793a29:41489 {}] master.HMaster(1795): Stopping service threads
2024-11-22T05:38:19,686 INFO  [M:0;82891a793a29:41489 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-22T05:38:19,686 INFO  [M:0;82891a793a29:41489 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T05:38:19,686 INFO  [M:0;82891a793a29:41489 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-22T05:38:19,686 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-22T05:38:19,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-22T05:38:19,698 DEBUG [M:0;82891a793a29:41489 {}] zookeeper.ZKUtil(347): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-22T05:38:19,699 WARN  [M:0;82891a793a29:41489 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-22T05:38:19,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T05:38:19,699 INFO  [M:0;82891a793a29:41489 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/.lastflushedseqids
2024-11-22T05:38:19,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741838_1014 (size=99)
2024-11-22T05:38:19,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741838_1014 (size=99)
2024-11-22T05:38:19,709 INFO  [M:0;82891a793a29:41489 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-22T05:38:19,710 INFO  [M:0;82891a793a29:41489 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-22T05:38:19,710 DEBUG [M:0;82891a793a29:41489 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T05:38:19,710 INFO  [M:0;82891a793a29:41489 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:38:19,710 DEBUG [M:0;82891a793a29:41489 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:38:19,710 DEBUG [M:0;82891a793a29:41489 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T05:38:19,710 DEBUG [M:0;82891a793a29:41489 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:38:19,710 INFO  [M:0;82891a793a29:41489 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-22T05:38:19,727 DEBUG [M:0;82891a793a29:41489 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8fac12fcdad14a57a4bdd6ded36dd26a is 82, key is hbase:meta,,1/info:regioninfo/1732253899237/Put/seqid=0
2024-11-22T05:38:19,731 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741839_1015 (size=5672)
2024-11-22T05:38:19,731 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741839_1015 (size=5672)
2024-11-22T05:38:19,731 INFO  [M:0;82891a793a29:41489 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8fac12fcdad14a57a4bdd6ded36dd26a
2024-11-22T05:38:19,747 DEBUG [M:0;82891a793a29:41489 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/91975eb7aff5412da40c3ff598a4442f is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732253899296/Put/seqid=0
2024-11-22T05:38:19,751 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741840_1016 (size=5275)
2024-11-22T05:38:19,751 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741840_1016 (size=5275)
2024-11-22T05:38:19,751 INFO  [M:0;82891a793a29:41489 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/91975eb7aff5412da40c3ff598a4442f
2024-11-22T05:38:19,768 DEBUG [M:0;82891a793a29:41489 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5dddf8e911a2443ebd07663693e16775 is 69, key is 82891a793a29,33869,1732253898160/rs:state/1732253898727/Put/seqid=0
2024-11-22T05:38:19,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741841_1017 (size=5156)
2024-11-22T05:38:19,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741841_1017 (size=5156)
2024-11-22T05:38:19,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,773 INFO  [M:0;82891a793a29:41489 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5dddf8e911a2443ebd07663693e16775
2024-11-22T05:38:19,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:38:19,775 INFO  [RS:0;82891a793a29:33869 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T05:38:19,775 INFO  [RS:0;82891a793a29:33869 {}] regionserver.HRegionServer(1031): Exiting; stopping=82891a793a29,33869,1732253898160; zookeeper connection closed.
2024-11-22T05:38:19,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33869-0x101610ccfb40001, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:38:19,775 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@130a9340 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@130a9340
2024-11-22T05:38:19,775 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-22T05:38:19,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,35099,1732253704679/82891a793a29%2C35099%2C1732253704679.meta.1732253705835.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:19,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46791/user/jenkins/test-data/3b1f21d5-1659-d7d9-aadf-8c0efd1c5253/WALs/82891a793a29,33487,1732253706069/82891a793a29%2C33487%2C1732253706069.1732253706319
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T05:38:19,789 DEBUG [M:0;82891a793a29:41489 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7deb54a5ea20404da228eb5925836bf9 is 52, key is load_balancer_on/state:d/1732253899390/Put/seqid=0
2024-11-22T05:38:19,793 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741842_1018 (size=5056)
2024-11-22T05:38:19,793 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741842_1018 (size=5056)
2024-11-22T05:38:19,794 INFO  [M:0;82891a793a29:41489 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7deb54a5ea20404da228eb5925836bf9
2024-11-22T05:38:19,798 DEBUG [M:0;82891a793a29:41489 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8fac12fcdad14a57a4bdd6ded36dd26a as hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8fac12fcdad14a57a4bdd6ded36dd26a
2024-11-22T05:38:19,802 INFO  [M:0;82891a793a29:41489 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8fac12fcdad14a57a4bdd6ded36dd26a, entries=8, sequenceid=29, filesize=5.5 K
2024-11-22T05:38:19,803 DEBUG [M:0;82891a793a29:41489 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/91975eb7aff5412da40c3ff598a4442f as hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/91975eb7aff5412da40c3ff598a4442f
2024-11-22T05:38:19,809 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,809 INFO  [M:0;82891a793a29:41489 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/91975eb7aff5412da40c3ff598a4442f, entries=3, sequenceid=29, filesize=5.2 K
2024-11-22T05:38:19,810 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,810 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,810 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,810 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,810 DEBUG [M:0;82891a793a29:41489 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5dddf8e911a2443ebd07663693e16775 as hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5dddf8e911a2443ebd07663693e16775
2024-11-22T05:38:19,811 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,817 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,817 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,818 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,818 INFO  [M:0;82891a793a29:41489 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5dddf8e911a2443ebd07663693e16775, entries=1, sequenceid=29, filesize=5.0 K
2024-11-22T05:38:19,819 DEBUG [M:0;82891a793a29:41489 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7deb54a5ea20404da228eb5925836bf9 as hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/7deb54a5ea20404da228eb5925836bf9
2024-11-22T05:38:19,821 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:19,840 INFO  [M:0;82891a793a29:41489 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46055/user/jenkins/test-data/c92050ea-7250-1188-af67-2e5a730853d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/7deb54a5ea20404da228eb5925836bf9, entries=1, sequenceid=29, filesize=4.9 K
2024-11-22T05:38:19,842 INFO  [M:0;82891a793a29:41489 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 131ms, sequenceid=29, compaction requested=false
2024-11-22T05:38:19,847 INFO  [M:0;82891a793a29:41489 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T05:38:19,847 DEBUG [M:0;82891a793a29:41489 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732253899710Disabling compacts and flushes for region at 1732253899710Disabling writes for close at 1732253899710Obtaining lock to block concurrent updates at 1732253899710Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732253899710Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732253899711 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732253899712 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732253899712Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732253899727 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732253899727Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732253899735 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732253899747 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732253899747Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732253899755 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732253899767 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732253899767Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732253899776 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732253899789 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732253899789Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39a2784f: reopening flushed file at 1732253899797 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@261eeef3: reopening flushed file at 1732253899802 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@664834fe: reopening flushed file at 1732253899809 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4be5177f: reopening flushed file at 1732253899818 (+9 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 131ms, sequenceid=29, compaction requested=false at 1732253899842 (+24 ms)Writing region close event to WAL at 1732253899847 (+5 ms)Closed at 1732253899847
2024-11-22T05:38:19,848 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,848 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,848 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,848 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,848 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T05:38:19,851 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36843 is added to blk_1073741830_1006 (size=10311)
2024-11-22T05:38:19,852 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741830_1006 (size=10311)
2024-11-22T05:38:20,154 DEBUG [master/82891a793a29:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=9, created chunk count=9, reused chunk count=79, reuseRatio=89.77%
2024-11-22T05:38:20,154 DEBUG [master/82891a793a29:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-11-22T05:38:20,253 INFO  [M:0;82891a793a29:41489 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-22T05:38:20,253 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T05:38:20,253 INFO  [M:0;82891a793a29:41489 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41489
2024-11-22T05:38:20,253 INFO  [M:0;82891a793a29:41489 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T05:38:20,332 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T05:38:20,333 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,335 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,335 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,335 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,336 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,337 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,337 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,337 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,362 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,362 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,362 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,362 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,362 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,363 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,366 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,366 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,367 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,369 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T05:38:20,407 INFO  [M:0;82891a793a29:41489 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T05:38:20,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:38:20,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41489-0x101610ccfb40000, quorum=127.0.0.1:59322, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T05:38:20,409 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2ea5aa4c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:38:20,410 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@59622fd9{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:38:20,410 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:38:20,410 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1ba1b0df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:38:20,410 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@242dc355{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/hadoop.log.dir/,STOPPED}
2024-11-22T05:38:20,411 WARN  [BP-220569439-172.17.0.2-1732253895346 heartbeating to localhost/127.0.0.1:46055 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:38:20,411 WARN  [BP-220569439-172.17.0.2-1732253895346 heartbeating to localhost/127.0.0.1:46055 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-220569439-172.17.0.2-1732253895346 (Datanode Uuid aa18fedd-173b-405c-aae4-8777c145bda0) service to localhost/127.0.0.1:46055
2024-11-22T05:38:20,411 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:38:20,412 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:38:20,412 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:38:20,412 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/cluster_0b2e7f0c-4f5f-a5c0-e78d-41c08b25443d/data/data4/current/BP-220569439-172.17.0.2-1732253895346 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:38:20,412 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/cluster_0b2e7f0c-4f5f-a5c0-e78d-41c08b25443d/data/data3/current/BP-220569439-172.17.0.2-1732253895346 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:38:20,413 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5dba136e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T05:38:20,414 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e2fef96{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:38:20,414 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:38:20,414 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@512a083f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:38:20,414 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@19f4d4a5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/hadoop.log.dir/,STOPPED}
2024-11-22T05:38:20,415 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T05:38:20,415 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T05:38:20,415 WARN  [BP-220569439-172.17.0.2-1732253895346 heartbeating to localhost/127.0.0.1:46055 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T05:38:20,415 WARN  [BP-220569439-172.17.0.2-1732253895346 heartbeating to localhost/127.0.0.1:46055 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-220569439-172.17.0.2-1732253895346 (Datanode Uuid 227a8f47-7bd0-41e9-93f2-524fc6c8cf3b) service to localhost/127.0.0.1:46055
2024-11-22T05:38:20,415 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/cluster_0b2e7f0c-4f5f-a5c0-e78d-41c08b25443d/data/data1/current/BP-220569439-172.17.0.2-1732253895346 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:38:20,416 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/cluster_0b2e7f0c-4f5f-a5c0-e78d-41c08b25443d/data/data2/current/BP-220569439-172.17.0.2-1732253895346 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T05:38:20,416 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T05:38:20,420 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4edfb46d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T05:38:20,421 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7d6e4d22{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T05:38:20,421 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T05:38:20,421 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20042785{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T05:38:20,421 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b01e479{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d5fcf089-52d3-b8b9-b97e-373e37685114/hadoop.log.dir/,STOPPED}
2024-11-22T05:38:20,426 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-22T05:38:20,440 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-22T05:38:20,447 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=269 (was 231)
Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46055
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46055 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46055 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46055 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46055
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:46055
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:46055
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46055
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=534 (was 518) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=161 (was 158) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7778 (was 7882)